MMF-935 experiment ES resilience of user creation

server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java

@@ -135,7 +135,7 @@ public class ComputeEngineContainerImplTest {
     assertThat(picoContainer.getParent().getParent().getParent().getComponentAdapters()).hasSize(
       COMPONENTS_IN_LEVEL_1_AT_CONSTRUCTION
         + 24 // level 1
-        + 46 // content of DaoModule
+        + 47 // content of DaoModule
         + 3 // content of EsSearchModule
         + 58 // content of CorePropertyDefinitions
     );

server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java

@@ -236,21 +236,6 @@ public class DatabaseUtils {
     }
   }
 
-  /**
-   * @deprecated replaced by Commons Land {@code StringUtils.repeat(sql, separator, count)
-   */
-  @Deprecated
-  public static String repeatCondition(String sql, int count, String separator) {
-    StringBuilder sb = new StringBuilder();
-    for (int i = 0; i < count; i++) {
-      sb.append(sql);
-      if (i < count - 1) {
-        sb.append(" ").append(separator).append(" ");
-      }
-    }
-    return sb.toString();
-  }
-
   /**
    * Logback does not log exceptions associated to {@link java.sql.SQLException#getNextException()}.
    * See http://jira.qos.ch/browse/LOGBACK-775

server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java

@@ -54,6 +54,7 @@ public final class SqTables {
     "ce_scanner_context",
     "default_qprofiles",
     "duplications_index",
+    "es_queue",
     "events",
     "file_sources",
     "groups",

server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl

@@ -658,3 +658,12 @@ CREATE TABLE "WEBHOOK_DELIVERIES" (
 CREATE UNIQUE INDEX "PK_WEBHOOK_DELIVERIES" ON "WEBHOOK_DELIVERIES" ("UUID");
 CREATE INDEX "COMPONENT_UUID" ON "WEBHOOK_DELIVERIES" ("COMPONENT_UUID");
 CREATE INDEX "CE_TASK_UUID" ON "WEBHOOK_DELIVERIES" ("CE_TASK_UUID");
+
+CREATE TABLE "ES_QUEUE" (
+  "UUID" VARCHAR(40) NOT NULL PRIMARY KEY,
+  "DOC_TYPE" VARCHAR(40) NOT NULL,
+  "DOC_UUID" VARCHAR(255) NOT NULL,
+  "CREATED_AT" BIGINT NOT NULL
+);
+CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID");
+CREATE INDEX "ES_QUEUE_CREATED_AT" ON "ES_QUEUE" ("CREATED_AT");

server/sonar-db-core/src/test/java/org/sonar/db/DatabaseUtilsTest.java

@@ -244,12 +244,6 @@ public class DatabaseUtilsTest {
     return sql;
   }
 
-  @Test
-  public void repeatCondition() {
-    assertThat(DatabaseUtils.repeatCondition("uuid=?", 1, "or")).isEqualTo("uuid=?");
-    assertThat(DatabaseUtils.repeatCondition("uuid=?", 3, "or")).isEqualTo("uuid=? or uuid=? or uuid=?");
-  }
-
   @Test
   public void executeLargeInputs() {
     List<Integer> inputs = newArrayList();

server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java

@@ -35,6 +35,7 @@ import org.sonar.db.duplication.DuplicationDao;
 import org.sonar.db.event.EventDao;
 import org.sonar.db.issue.IssueChangeDao;
 import org.sonar.db.issue.IssueDao;
+import org.sonar.db.es.EsQueueDao;
 import org.sonar.db.loadedtemplate.LoadedTemplateDao;
 import org.sonar.db.measure.MeasureDao;
 import org.sonar.db.measure.custom.CustomMeasureDao;
@@ -86,6 +87,7 @@ public class DaoModule extends Module {
     CustomMeasureDao.class,
     DefaultQProfileDao.class,
     DuplicationDao.class,
+    EsQueueDao.class,
     EventDao.class,
     FileSourceDao.class,
     GroupDao.class,

server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java

@@ -34,6 +34,7 @@ import org.sonar.db.duplication.DuplicationDao;
 import org.sonar.db.event.EventDao;
 import org.sonar.db.issue.IssueChangeDao;
 import org.sonar.db.issue.IssueDao;
+import org.sonar.db.es.EsQueueDao;
 import org.sonar.db.loadedtemplate.LoadedTemplateDao;
 import org.sonar.db.measure.MeasureDao;
 import org.sonar.db.measure.custom.CustomMeasureDao;
@@ -118,6 +119,7 @@ public class DbClient {
   private final UserPermissionDao userPermissionDao;
   private final WebhookDeliveryDao webhookDeliveryDao;
   private final DefaultQProfileDao defaultQProfileDao;
+  private final EsQueueDao esQueueDao;
 
   public DbClient(Database database, MyBatis myBatis, Dao... daos) {
     this.database = database;
@@ -173,6 +175,7 @@ public class DbClient {
     userPermissionDao = getDao(map, UserPermissionDao.class);
     webhookDeliveryDao = getDao(map, WebhookDeliveryDao.class);
     defaultQProfileDao = getDao(map, DefaultQProfileDao.class);
+    esQueueDao = getDao(map, EsQueueDao.class);
   }
 
   public DbSession openSession(boolean batch) {
@@ -367,6 +370,10 @@ public class DbClient {
     return defaultQProfileDao;
   }
 
+  public EsQueueDao esQueueDao() {
+    return esQueueDao;
+  }
+
   protected <K extends Dao> K getDao(Map<Class, Dao> map, Class<K> clazz) {
     return (K) map.get(clazz);
   }

server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java

@@ -55,6 +55,7 @@ import org.sonar.db.issue.IssueChangeDto;
 import org.sonar.db.issue.IssueChangeMapper;
 import org.sonar.db.issue.IssueDto;
 import org.sonar.db.issue.IssueMapper;
+import org.sonar.db.es.EsQueueMapper;
 import org.sonar.db.loadedtemplate.LoadedTemplateDto;
 import org.sonar.db.loadedtemplate.LoadedTemplateMapper;
 import org.sonar.db.measure.MeasureDto;
@@ -197,6 +198,7 @@ public class MyBatis implements Startable {
       CustomMeasureMapper.class,
       DefaultQProfileMapper.class,
       DuplicationMapper.class,
+      EsQueueMapper.class,
       EventMapper.class,
       FileSourceMapper.class,
       GroupMapper.class,

server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDao.java

@@ -0,0 +1,79 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.es;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import org.sonar.api.utils.System2;
+import org.sonar.core.util.UuidFactory;
+import org.sonar.db.Dao;
+import org.sonar.db.DbSession;
+
+import static java.util.Collections.singletonList;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+import static org.sonar.db.DatabaseUtils.executeLargeUpdates;
+
+public class EsQueueDao implements Dao {
+
+  private final System2 system2;
+  private final UuidFactory uuidFactory;
+
+  public EsQueueDao(System2 system2, UuidFactory uuidFactory) {
+    this.system2 = system2;
+    this.uuidFactory = uuidFactory;
+  }
+
+  public EsQueueDto insert(DbSession dbSession, EsQueueDto item) {
+    insert(dbSession, singletonList(item));
+    return item;
+  }
+
+  public Collection<EsQueueDto> insert(DbSession dbSession, Collection<EsQueueDto> items) {
+    long now = system2.now();
+    EsQueueMapper mapper = mapper(dbSession);
+    items.forEach(item -> {
+      item.setUuid(uuidFactory.create());
+      mapper.insert(item, now);
+    });
+    return items;
+  }
+
+  public void delete(DbSession dbSession, EsQueueDto item) {
+    delete(dbSession, singletonList(item));
+  }
+
+  public void delete(DbSession dbSession, Collection<EsQueueDto> items) {
+    EsQueueMapper mapper = mapper(dbSession);
+    List<String> uuids = items.stream()
+      .map(EsQueueDto::getUuid)
+      .filter(Objects::nonNull)
+      .collect(toArrayList(items.size()));
+    executeLargeUpdates(uuids, mapper::delete);
+  }
+
+  public Collection<EsQueueDto> selectForRecovery(DbSession dbSession, long beforeDate, long limit) {
+    return mapper(dbSession).selectForRecovery(beforeDate, limit);
+  }
+
+  private static EsQueueMapper mapper(DbSession dbSession) {
+    return dbSession.getMapper(EsQueueMapper.class);
+  }
+}

server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java

@@ -0,0 +1,72 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.es;
+
+public final class EsQueueDto {
+
+  public enum Type {
+    USER
+  }
+
+  private String uuid;
+  private Type docType;
+  private String docUuid;
+
+  public String getUuid() {
+    return uuid;
+  }
+
+  EsQueueDto setUuid(String uuid) {
+    this.uuid = uuid;
+    return this;
+  }
+
+  public Type getDocType() {
+    return docType;
+  }
+
+  private EsQueueDto setDocType(Type t) {
+    this.docType = t;
+    return this;
+  }
+
+  public String getDocUuid() {
+    return docUuid;
+  }
+
+  private EsQueueDto setDocUuid(String s) {
+    this.docUuid = s;
+    return this;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder("EsQueueDto{");
+    sb.append("uuid='").append(uuid).append('\'');
+    sb.append(", docType=").append(docType);
+    sb.append(", docUuid='").append(docUuid).append('\'');
+    sb.append('}');
+    return sb.toString();
+  }
+
+  public static EsQueueDto create(Type docType, String docUuid) {
+    return new EsQueueDto().setDocType(docType).setDocUuid(docUuid);
+  }
+}

server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueMapper.java

@@ -0,0 +1,33 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.es;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.ibatis.annotations.Param;
+
+public interface EsQueueMapper {
+
+  void insert(@Param("dto") EsQueueDto dto, @Param("now") long now);
+
+  void delete(@Param("uuids") List<String> uuids);
+
+  Collection<EsQueueDto> selectForRecovery(@Param("beforeDate") long beforeDate, @Param("limit") long limit);
+}

server/sonar-db-dao/src/main/java/org/sonar/db/es/package-info.java

@@ -0,0 +1,24 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.db.es;
+
+import javax.annotation.ParametersAreNonnullByDefault;
+

server/sonar-db-dao/src/main/java/org/sonar/db/organization/OrganizationMemberDao.java

@@ -20,6 +20,7 @@
 
 package org.sonar.db.organization;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -70,7 +71,7 @@ public class OrganizationMemberDao implements Dao {
    *
    * @param loginOrganizationConsumer {@link BiConsumer}<String,String> (login, organization uuid)
    */
-  public void selectForUserIndexing(DbSession dbSession, List<String> logins, BiConsumer<String, String> loginOrganizationConsumer) {
+  public void selectForUserIndexing(DbSession dbSession, Collection<String> logins, BiConsumer<String, String> loginOrganizationConsumer) {
     executeLargeInputsWithoutOutput(logins, list -> mapper(dbSession).selectForIndexing(list)
       .forEach(row -> loginOrganizationConsumer.accept(row.get("login"), row.get("organizationUuid"))));
   }

server/sonar-db-dao/src/main/java/org/sonar/db/property/PropertiesDao.java

@@ -33,12 +33,12 @@ import javax.annotation.Nullable;
 import org.sonar.api.resources.Scopes;
 import org.sonar.api.utils.System2;
 import org.sonar.db.Dao;
-import org.sonar.db.DatabaseUtils;
 import org.sonar.db.DbSession;
 import org.sonar.db.MyBatis;
 import org.sonar.db.WildcardPosition;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.commons.lang.StringUtils.repeat;
 import static org.sonar.db.DaoDatabaseUtils.buildLikeValue;
 import static org.sonar.db.DatabaseUtils.executeLargeInputs;
 import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput;
@@ -89,7 +89,7 @@ public class PropertiesDao implements Dao {
     String sql = "SELECT count(1) FROM properties pp " +
       "left outer join projects pj on pp.resource_id = pj.id " +
       "where pp.user_id is not null and (pp.resource_id is null or pj.uuid=?) " +
-      "and (" + DatabaseUtils.repeatCondition("pp.prop_key like ?", dispatcherKeys.size(), "or") + ")";
+      "and (" + repeat("pp.prop_key like ?", " or ", dispatcherKeys.size()) + ")";
     PreparedStatement res = connection.prepareStatement(sql);
     res.setString(1, projectUuid);
     int index = 2;
@@ -282,7 +282,7 @@ public class PropertiesDao implements Dao {
     executeLargeInputsWithoutOutput(ids, list -> getMapper(dbSession).deleteByIds(list));
   }
 
-  public void deleteByKeyAndValue(DbSession dbSession, String key, String value){
+  public void deleteByKeyAndValue(DbSession dbSession, String key, String value) {
     getMapper(dbSession).deleteByKeyAndValue(key, value);
   }
 

server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java

@@ -19,13 +19,15 @@
  */
 package org.sonar.db.user;
 
-import com.google.common.base.Function;
-import com.google.common.base.Predicates;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
 import org.sonar.api.user.UserQuery;
@@ -34,8 +36,8 @@ import org.sonar.db.Dao;
 import org.sonar.db.DbSession;
 import org.sonar.db.RowNotFoundException;
 
-import static com.google.common.collect.FluentIterable.from;
 import static org.sonar.db.DatabaseUtils.executeLargeInputs;
+import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput;
 
 public class UserDao implements Dao {
 
@@ -82,7 +84,10 @@ public class UserDao implements Dao {
    */
   public List<UserDto> selectByOrderedLogins(DbSession session, Collection<String> logins) {
     List<UserDto> unordered = selectByLogins(session, logins);
-    return from(logins).transform(new LoginToUser(unordered)).filter(Predicates.notNull()).toList();
+    return logins.stream()
+      .map(new LoginToUser(unordered))
+      .filter(Objects::nonNull)
+      .collect(Collectors.toList());
   }
 
   public List<UserDto> selectUsers(DbSession dbSession, UserQuery query) {
@@ -94,12 +99,17 @@ public class UserDao implements Dao {
   }
 
   public UserDto insert(DbSession session, UserDto dto) {
-    mapper(session).insert(dto);
+    long now = system2.now();
+    mapper(session).insert(dto, now);
+    dto.setCreatedAt(now);
+    dto.setUpdatedAt(now);
     return dto;
   }
 
   public UserDto update(DbSession session, UserDto dto) {
-    mapper(session).update(dto);
+    long now = system2.now();
+    mapper(session).update(dto, now);
+    dto.setUpdatedAt(now);
     return dto;
   }
 
@@ -107,8 +117,8 @@ public class UserDao implements Dao {
     mapper(session).setRoot(login, root, system2.now());
   }
 
-  public void deactivateUserById(DbSession dbSession, int userId) {
-    mapper(dbSession).deactivateUser(userId, system2.now());
+  public void deactivateUser(DbSession dbSession, UserDto user) {
+    mapper(dbSession).deactivateUser(user.getLogin(), system2.now());
   }
 
   @CheckForNull
@@ -140,6 +150,22 @@ public class UserDao implements Dao {
     return mapper(dbSession).countByEmail(email.toLowerCase(Locale.ENGLISH)) > 0;
   }
 
+  public void scrollByLogins(DbSession dbSession, Collection<String> logins, Consumer<UserDto> consumer) {
+    UserMapper mapper = mapper(dbSession);
+
+    executeLargeInputsWithoutOutput(logins,
+      pageOfLogins -> mapper
+        .selectByLogins(pageOfLogins)
+        .forEach(consumer));
+  }
+
+  public void scrollAll(DbSession dbSession, Consumer<UserDto> consumer) {
+    mapper(dbSession).scrollAll(context -> {
+      UserDto user = (UserDto) context.getResultObject();
+      consumer.accept(user);
+    });
+  }
+
   private static UserMapper mapper(DbSession session) {
     return session.getMapper(UserMapper.class);
   }

server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDto.java

@@ -189,7 +189,7 @@ public class UserDto {
     return createdAt;
   }
 
-  public UserDto setCreatedAt(Long createdAt) {
+  UserDto setCreatedAt(long createdAt) {
     this.createdAt = createdAt;
     return this;
   }
@@ -198,7 +198,7 @@ public class UserDto {
     return updatedAt;
   }
 
-  public UserDto setUpdatedAt(Long updatedAt) {
+  UserDto setUpdatedAt(long updatedAt) {
     this.updatedAt = updatedAt;
     return this;
   }

server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java

@@ -22,6 +22,7 @@ package org.sonar.db.user;
 import java.util.List;
 import javax.annotation.CheckForNull;
 import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.session.ResultHandler;
 import org.sonar.api.user.UserQuery;
 
 public interface UserMapper {
@@ -51,6 +52,8 @@ public interface UserMapper {
 
   List<UserDto> selectByIds(@Param("ids") List<Integer> ids);
 
+  void scrollAll(ResultHandler handler);
+
   long countByEmail(String email);
 
   /**
@@ -58,14 +61,12 @@ public interface UserMapper {
    */
   long countRootUsersButLogin(@Param("login") String login);
 
-  void insert(UserDto userDto);
+  void insert(@Param("user") UserDto userDto, @Param("now") long now);
 
-  void update(UserDto userDto);
+  void update(@Param("user") UserDto userDto, @Param("now") long now);
 
   void setRoot(@Param("login") String login, @Param("root") boolean root, @Param("now") long now);
 
-  void deleteOrganisationMembership(int userId);
-
-  void deactivateUser(@Param("id") int userId, @Param("now") long now);
+  void deactivateUser(@Param("login") String login, @Param("now") long now);
 
 }

server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="org.sonar.db.es.EsQueueMapper">
+
+  <sql id="esQueueColumns">
+    uuid,
+    doc_type as docType,
+    doc_uuid as docUuid,
+    created_at as createdAt
+  </sql>
+
+  <insert id="insert" parameterType="map" useGeneratedKeys="false">
+    insert into es_queue (
+      uuid,
+      doc_type,
+      doc_uuid,
+      created_at
+    ) values (
+      #{dto.uuid, jdbcType=VARCHAR},
+      #{dto.docType, jdbcType=VARCHAR},
+      #{dto.docUuid, jdbcType=VARCHAR},
+      #{now, jdbcType=BIGINT}
+    )
+  </insert>
+
+  <delete id="delete" parameterType="string">
+    delete from es_queue
+    where uuid in
+      <foreach item="uuid" collection="uuids" open="(" separator="," close=")">
+        #{uuid, jdbcType=VARCHAR}
+      </foreach>
+  </delete>
+
+  <select id="selectForRecovery" parameterType="map" resultType="org.sonar.db.es.EsQueueDto">
+    select <include refid="esQueueColumns" />
+    from es_queue
+    where
+    created_at &lt;= #{beforeDate, jdbcType=BIGINT}
+    order by created_at desc
+    limit #{limit, jdbcType=INTEGER}
+  </select>
+
+  <select id="selectForRecovery" parameterType="map" resultType="org.sonar.db.es.EsQueueDto" databaseId="oracle">
+    select * from (
+    select rownum as rn, t.* from (
+      select <include refid="esQueueColumns" />
+      from es_queue
+      where
+      created_at &lt;= #{beforeDate, jdbcType=BIGINT}
+      order by created_at desc
+    ) t
+    ) t
+    where
+    t.rn &lt;= #{limit, jdbcType=INTEGER}
+  </select>
+
+  <select id="selectForRecovery" parameterType="map" resultType="org.sonar.db.es.EsQueueDto" databaseId="mssql">
+    select top(#{limit, jdbcType=INTEGER}) <include refid="esQueueColumns" />
+    from es_queue
+    where
+    created_at &lt;= #{beforeDate, jdbcType=BIGINT}
+    order by created_at desc
+  </select>
+
+</mapper>
+

server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml

@@ -62,6 +62,11 @@
     </foreach>
   </select>
 
+  <select id="scrollAll" resultType="User" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    select <include refid="userColumns"/>
+    from users u
+  </select>
+
   <select id="selectByIds" parameterType="string" resultType="User">
     SELECT
     <include refid="userColumns"/>
@@ -116,35 +121,31 @@
       and u.login &lt;&gt; #{login}
   </select>
 
-  <delete id="deleteOrganisationMembership" parameterType="int">
-    DELETE FROM organization_members WHERE user_id=#{id,jdbcType=BIGINT}
-  </delete>
-
   <update id="deactivateUser" parameterType="map">
-    UPDATE users SET
-      active=${_false},
-      email=null,
-      scm_accounts=null,
-      external_identity=null,
-      external_identity_provider=null,
-      salt=null,
-      crypted_password=null,
-      updated_at=#{now,jdbcType=BIGINT}
-    WHERE
-      id=#{id,jdbcType=INTEGER}
+    update users set
+      active = ${_false},
+      email = null,
+      scm_accounts = null,
+      external_identity = null,
+      external_identity_provider = null,
+      salt = null,
+      crypted_password = null,
+      updated_at = #{now, jdbcType=BIGINT}
+    where
+      login = #{login, jdbcType=VARCHAR}
   </update>
 
   <update id="setRoot">
     update users set
-      is_root = #{root,jdbcType=BOOLEAN},
-      updated_at=#{now,jdbcType=BIGINT}
+      is_root = #{root, jdbcType=BOOLEAN},
+      updated_at = #{now, jdbcType=BIGINT}
     where
-      login = #{login,jdbcType=VARCHAR}
+      login = #{login, jdbcType=VARCHAR}
       and active = ${_true}
   </update>
 
-  <insert id="insert" parameterType="User" keyColumn="id" useGeneratedKeys="true" keyProperty="id">
-    INSERT INTO users (
+  <insert id="insert" parameterType="map" keyColumn="id" useGeneratedKeys="true" keyProperty="user.id">
+    insert into users (
       login,
       name,
       email,
@@ -159,40 +160,39 @@
       onboarded,
       created_at,
       updated_at
-    )
-    VALUES (
-      #{login,jdbcType=VARCHAR},
-      #{name,jdbcType=VARCHAR},
-      #{email,jdbcType=VARCHAR},
-      #{active,jdbcType=BOOLEAN},
-      #{scmAccounts,jdbcType=VARCHAR},
-      #{externalIdentity,jdbcType=VARCHAR},
-      #{externalIdentityProvider,jdbcType=VARCHAR},
-      #{local,jdbcType=BOOLEAN},
-      #{salt,jdbcType=VARCHAR},
-      #{cryptedPassword,jdbcType=VARCHAR},
-      #{root,jdbcType=BOOLEAN},
-      #{onboarded,jdbcType=BOOLEAN},
-      #{createdAt,jdbcType=BIGINT},
-      #{updatedAt,jdbcType=BIGINT}
+    ) values (
+      #{user.login,jdbcType=VARCHAR},
+      #{user.name,jdbcType=VARCHAR},
+      #{user.email,jdbcType=VARCHAR},
+      #{user.active,jdbcType=BOOLEAN},
+      #{user.scmAccounts,jdbcType=VARCHAR},
+      #{user.externalIdentity,jdbcType=VARCHAR},
+      #{user.externalIdentityProvider,jdbcType=VARCHAR},
+      #{user.local,jdbcType=BOOLEAN},
+      #{user.salt,jdbcType=VARCHAR},
+      #{user.cryptedPassword,jdbcType=VARCHAR},
+      #{user.root,jdbcType=BOOLEAN},
+      #{user.onboarded,jdbcType=BOOLEAN},
+      #{now,jdbcType=BIGINT},
+      #{now,jdbcType=BIGINT}
     )
   </insert>
 
-  <insert id="update" parameterType="User" useGeneratedKeys="false">
-    UPDATE users set
-      name=#{name,jdbcType=VARCHAR},
-      email=#{email,jdbcType=VARCHAR},
-      active=#{active,jdbcType=BOOLEAN},
-      scm_accounts=#{scmAccounts,jdbcType=VARCHAR},
-      external_identity=#{externalIdentity,jdbcType=VARCHAR},
-      external_identity_provider=#{externalIdentityProvider,jdbcType=VARCHAR},
-      user_local=#{local,jdbcType=BOOLEAN},
-      onboarded=#{onboarded,jdbcType=BOOLEAN},
-      salt=#{salt,jdbcType=VARCHAR},
-      crypted_password=#{cryptedPassword,jdbcType=BIGINT},
-      updated_at=#{updatedAt,jdbcType=BIGINT}
-    WHERE
-      login = #{login,jdbcType=VARCHAR}
-  </insert>
+  <update id="update" parameterType="map">
+    update users set
+      name = #{user.name, jdbcType=VARCHAR},
+      email = #{user.email, jdbcType=VARCHAR},
+      active = #{user.active, jdbcType=BOOLEAN},
+      scm_accounts = #{user.scmAccounts, jdbcType=VARCHAR},
+      external_identity = #{user.externalIdentity, jdbcType=VARCHAR},
+      external_identity_provider = #{user.externalIdentityProvider, jdbcType=VARCHAR},
+      user_local = #{user.local, jdbcType=BOOLEAN},
+      onboarded = #{user.onboarded, jdbcType=BOOLEAN},
+      salt = #{user.salt, jdbcType=VARCHAR},
+      crypted_password = #{user.cryptedPassword, jdbcType=BIGINT},
+      updated_at = #{now, jdbcType=BIGINT}
+    where
+      login = #{user.login, jdbcType=VARCHAR}
+  </update>
 
 </mapper>

server/sonar-db-dao/src/test/java/org/sonar/db/DaoModuleTest.java

@@ -29,6 +29,6 @@ public class DaoModuleTest {
   public void verify_count_of_added_components() {
     ComponentContainer container = new ComponentContainer();
     new DaoModule().configure(container);
-    assertThat(container.size()).isEqualTo(2 + 46);
+    assertThat(container.size()).isEqualTo(2 + 47);
   }
 }

server/sonar-db-dao/src/test/java/org/sonar/db/es/EsQueueDaoTest.java

@@ -0,0 +1,129 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.es;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.stream.IntStream;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.utils.internal.TestSystem2;
+import org.sonar.core.util.UuidFactoryFast;
+import org.sonar.db.DbSession;
+import org.sonar.db.DbTester;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class EsQueueDaoTest {
+
+  private static final int LIMIT = 10;
+  private static TestSystem2 system2 = new TestSystem2().setNow(1_000);
+
+  @Rule
+  public DbTester dbTester = DbTester.create(system2);
+
+  private DbSession dbSession = dbTester.getSession();
+  private EsQueueDao underTest = dbTester.getDbClient().esQueueDao();
+
+  @Test
+  public void insert_data()  {
+    int nbOfInsert = 10 + new Random().nextInt(20);
+    List<EsQueueDto> esQueueDtos = new ArrayList<>();
+    IntStream.rangeClosed(1, nbOfInsert).forEach(
+      i -> esQueueDtos.add(EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()))
+    );
+    underTest.insert(dbSession, esQueueDtos);
+
+    assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(nbOfInsert);
+  }
+
+  @Test
+  public void delete_unknown_EsQueueDto_does_not_throw_exception() {
+    int nbOfInsert = 10 + new Random().nextInt(20);
+    List<EsQueueDto> esQueueDtos = new ArrayList<>();
+    IntStream.rangeClosed(1, nbOfInsert).forEach(
+      i -> esQueueDtos.add(EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()))
+    );
+    underTest.insert(dbSession, esQueueDtos);
+
+    underTest.delete(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()));
+
+    assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(nbOfInsert);
+  }
+
+  @Test
+  public void delete_EsQueueDto_does_not_throw_exception() {
+    int nbOfInsert = 10 + new Random().nextInt(20);
+    List<EsQueueDto> esQueueDtos = new ArrayList<>();
+    IntStream.rangeClosed(1, nbOfInsert).forEach(
+      i -> esQueueDtos.add(EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()))
+    );
+    underTest.insert(dbSession, esQueueDtos);
+    assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(nbOfInsert);
+
+    underTest.delete(dbSession, esQueueDtos);
+
+    assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(0);
+  }
+
+  @Test
+  public void selectForRecovery_must_return_limit_when_there_are_more_rows()  {
+    system2.setNow(1_000L);
+    EsQueueDto i1 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()));
+    system2.setNow(1_001L);
+    EsQueueDto i2 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()));
+    system2.setNow(1_002L);
+    EsQueueDto i3 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()));
+
+    assertThat(underTest.selectForRecovery(dbSession, 2_000, 1))
+      .extracting(EsQueueDto::getUuid)
+      .containsExactly(i3.getUuid());
+
+    assertThat(underTest.selectForRecovery(dbSession, 2_000, 2))
+      .extracting(EsQueueDto::getUuid)
+      .containsExactly(i3.getUuid(), i2.getUuid());
+
+    assertThat(underTest.selectForRecovery(dbSession, 2_000, 10))
+      .extracting(EsQueueDto::getUuid)
+      .containsExactly(i3.getUuid(), i2.getUuid(), i1.getUuid());
+  }
+
+  @Test
+  public void selectForRecovery_returns_ordered_rows_created_before_date()  {
+    system2.setNow(1_000L);
+    EsQueueDto i1 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()));
+    system2.setNow(1_001L);
+    EsQueueDto i2 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()));
+    system2.setNow(1_002L);
+    EsQueueDto i3 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create()));
+
+    assertThat(underTest.selectForRecovery(dbSession, 999, LIMIT)).hasSize(0);
+    assertThat(underTest.selectForRecovery(dbSession, 1_000, LIMIT))
+      .extracting(EsQueueDto::getUuid)
+      .containsExactly(i1.getUuid());
+    assertThat(underTest.selectForRecovery(dbSession, 1_001, LIMIT))
+      .extracting(EsQueueDto::getUuid)
+      .containsExactly(i2.getUuid(), i1.getUuid());
+    assertThat(underTest.selectForRecovery(dbSession, 2_000, LIMIT))
+      .extracting(EsQueueDto::getUuid)
+      .containsExactly(i3.getUuid(), i2.getUuid(), i1.getUuid());
+  }
+}

server/sonar-db-dao/src/test/java/org/sonar/db/user/RootFlagAssertions.java

@@ -31,10 +31,6 @@ public class RootFlagAssertions {
     this.db = db;
   }
 
-  public void verifyUnchanged(UserDto user) {
-    verify(user, user.isRoot(), user.getUpdatedAt());
-  }
-
   public void verify(UserDto userDto, boolean root, long updatedAt) {
     Map<String, Object> row = db.selectFirst("select is_root as \"isRoot\", updated_at as \"updatedAt\" from users where login = '" + userDto.getLogin() + "'");
     Object isRoot = row.get("isRoot");

server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.db.user;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import org.junit.Before;
@@ -28,6 +29,7 @@ import org.junit.rules.ExpectedException;
 import org.sonar.api.user.UserQuery;
 import org.sonar.api.utils.DateUtils;
 import org.sonar.api.utils.System2;
+import org.sonar.db.DatabaseUtils;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
@@ -37,6 +39,7 @@ import static java.util.Arrays.asList;
 import static java.util.Collections.emptyList;
 import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.groups.Tuple.tuple;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -58,7 +61,7 @@ public class UserDaoTest {
   private UserDao underTest = db.getDbClient().userDao();
 
   @Before
-  public void setUp() throws Exception {
+  public void setUp() {
     when(system2.now()).thenReturn(NOW);
   }
 
@@ -337,23 +340,19 @@ public class UserDaoTest {
     assertThat(user.getExternalIdentityProvider()).isEqualTo("github");
     assertThat(user.isLocal()).isTrue();
     assertThat(user.isRoot()).isFalse();
-    assertThat(user.getCreatedAt()).isEqualTo(date);
-    assertThat(user.getUpdatedAt()).isEqualTo(date);
   }
 
   @Test
   public void update_user() {
-    UserDto existingUser = db.users().insertUser(user -> user
+    UserDto user = db.users().insertUser(u -> u
       .setLogin("john")
       .setName("John")
       .setEmail("jo@hn.com")
-      .setCreatedAt(1418215735482L)
-      .setUpdatedAt(1418215735482L)
       .setActive(true)
       .setLocal(true)
       .setOnboarded(false));
 
-    UserDto userDto = new UserDto()
+    UserDto userUpdate = new UserDto()
       .setId(1)
       .setLogin("john")
       .setName("John Doo")
@@ -365,38 +364,34 @@ public class UserDaoTest {
       .setCryptedPassword("abcde")
       .setExternalIdentity("johngithub")
       .setExternalIdentityProvider("github")
-      .setLocal(false)
-      .setUpdatedAt(1500000000000L);
-    underTest.update(db.getSession(), userDto);
-    db.getSession().commit();
-
-    UserDto user = underTest.selectUserById(db.getSession(), existingUser.getId());
-    assertThat(user).isNotNull();
-    assertThat(user.getId()).isEqualTo(existingUser.getId());
-    assertThat(user.getLogin()).isEqualTo("john");
-    assertThat(user.getName()).isEqualTo("John Doo");
-    assertThat(user.getEmail()).isEqualTo("jodoo@hn.com");
-    assertThat(user.isActive()).isFalse();
-    assertThat(user.isOnboarded()).isTrue();
-    assertThat(user.getScmAccounts()).isEqualTo(",jo.hn,john2,johndoo,");
-    assertThat(user.getSalt()).isEqualTo("12345");
-    assertThat(user.getCryptedPassword()).isEqualTo("abcde");
-    assertThat(user.getExternalIdentity()).isEqualTo("johngithub");
-    assertThat(user.getExternalIdentityProvider()).isEqualTo("github");
-    assertThat(user.isLocal()).isFalse();
-    assertThat(user.isRoot()).isFalse();
-    assertThat(user.getCreatedAt()).isEqualTo(1418215735482L);
-    assertThat(user.getUpdatedAt()).isEqualTo(1500000000000L);
-  }
-
-  @Test
-  public void deactivate_user() throws Exception {
-    UserDto user = newActiveUser();
+      .setLocal(false);
+    underTest.update(db.getSession(), userUpdate);
+
+    UserDto reloaded = underTest.selectByLogin(db.getSession(), user.getLogin());
+    assertThat(reloaded).isNotNull();
+    assertThat(reloaded.getId()).isEqualTo(user.getId());
+    assertThat(reloaded.getLogin()).isEqualTo(user.getLogin());
+    assertThat(reloaded.getName()).isEqualTo("John Doo");
+    assertThat(reloaded.getEmail()).isEqualTo("jodoo@hn.com");
+    assertThat(reloaded.isActive()).isFalse();
+    assertThat(reloaded.isOnboarded()).isTrue();
+    assertThat(reloaded.getScmAccounts()).isEqualTo(",jo.hn,john2,johndoo,");
+    assertThat(reloaded.getSalt()).isEqualTo("12345");
+    assertThat(reloaded.getCryptedPassword()).isEqualTo("abcde");
+    assertThat(reloaded.getExternalIdentity()).isEqualTo("johngithub");
+    assertThat(reloaded.getExternalIdentityProvider()).isEqualTo("github");
+    assertThat(reloaded.isLocal()).isFalse();
+    assertThat(reloaded.isRoot()).isFalse();
+  }
+
+  @Test
+  public void deactivate_user() {
+    UserDto user = insertActiveUser();
     insertUserGroup(user);
-    UserDto otherUser = newActiveUser();
+    UserDto otherUser = insertActiveUser();
     session.commit();
 
-    underTest.deactivateUserById(session, user.getId());
+    underTest.deactivateUser(session, user);
 
     UserDto userReloaded = underTest.selectUserById(session, user.getId());
     assertThat(userReloaded.isActive()).isFalse();
@@ -413,7 +408,7 @@ public class UserDaoTest {
 
   @Test
   public void does_not_fail_to_deactivate_missing_user() {
-    underTest.deactivateUserById(session, 123);
+    underTest.deactivateUser(session, UserTesting.newUserDto());
   }
 
   @Test
@@ -425,13 +420,11 @@ public class UserDaoTest {
       .setActive(true)
       .setScmAccounts("\nma\nmarius33\n")
       .setSalt("79bd6a8e79fb8c76ac8b121cc7e8e11ad1af8365")
-      .setCryptedPassword("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg")
-      .setCreatedAt(1418215735482L)
-      .setUpdatedAt(1418215735485L));
-    UserDto user2 = db.users().insertUser(user -> user.setLogin("sbrandhof"));
+      .setCryptedPassword("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg"));
+    UserDto user2 = db.users().insertUser();
     underTest.setRoot(session, user2.getLogin(), true);
 
-    UserDto dto = underTest.selectOrFailByLogin(session, "marius");
+    UserDto dto = underTest.selectOrFailByLogin(session, user1.getLogin());
     assertThat(dto.getId()).isEqualTo(user1.getId());
     assertThat(dto.getLogin()).isEqualTo("marius");
     assertThat(dto.getName()).isEqualTo("Marius");
@@ -441,10 +434,10 @@ public class UserDaoTest {
     assertThat(dto.getSalt()).isEqualTo("79bd6a8e79fb8c76ac8b121cc7e8e11ad1af8365");
     assertThat(dto.getCryptedPassword()).isEqualTo("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg");
     assertThat(dto.isRoot()).isFalse();
-    assertThat(dto.getCreatedAt()).isEqualTo(1418215735482L);
-    assertThat(dto.getUpdatedAt()).isEqualTo(1418215735485L);
+    assertThat(dto.getCreatedAt()).isEqualTo(user1.getCreatedAt());
+    assertThat(dto.getUpdatedAt()).isEqualTo(user1.getUpdatedAt());
 
-    dto = underTest.selectOrFailByLogin(session, "sbrandhof");
+    dto = underTest.selectOrFailByLogin(session, user2.getLogin());
     assertThat(dto.isRoot()).isTrue();
   }
 
@@ -490,8 +483,8 @@ public class UserDaoTest {
   }
 
   @Test
-  public void exists_by_email() throws Exception {
-    UserDto activeUser = newActiveUser();
+  public void exists_by_email() {
+    UserDto activeUser = insertActiveUser();
     UserDto disableUser = insertUser(false);
 
     assertThat(underTest.doesEmailExist(session, activeUser.getEmail())).isTrue();
@@ -507,8 +500,8 @@ public class UserDaoTest {
 
   @Test
   public void setRoot_set_root_flag_of_specified_user_to_specified_value_and_updates_udpateAt() {
-    String login = newActiveUser().getLogin();
-    UserDto otherUser = newActiveUser();
+    String login = insertActiveUser().getLogin();
+    UserDto otherUser = insertActiveUser();
     assertThat(underTest.selectByLogin(session, login).isRoot()).isEqualTo(false);
     assertThat(underTest.selectByLogin(session, otherUser.getLogin()).isRoot()).isEqualTo(false);
 
@@ -550,7 +543,7 @@ public class UserDaoTest {
     assertThat(underTest.selectByLogin(session, nonRootInactiveUser).isRoot()).isFalse();
 
     // create inactive root user
-    UserDto rootUser = newActiveUser();
+    UserDto rootUser = insertActiveUser();
     commit(() -> underTest.setRoot(session, rootUser.getLogin(), true));
     rootUser.setActive(false);
     commit(() -> underTest.update(session, rootUser));
@@ -562,12 +555,52 @@ public class UserDaoTest {
     assertThat(underTest.selectByLogin(session, inactiveRootUser.getLogin()).isRoot()).isTrue();
   }
 
+  @Test
+  public void scrollByLogins() {
+    UserDto u1 = insertUser(true);
+    UserDto u2 = insertUser(false);
+    UserDto u3 = insertUser(false);
+
+    List<UserDto> result = new ArrayList<>();
+    underTest.scrollByLogins(db.getSession(), asList(u2.getLogin(), u3.getLogin(), "does not exist"), result::add);
+
+    assertThat(result).extracting(UserDto::getLogin, UserDto::getName)
+      .containsExactlyInAnyOrder(tuple(u2.getLogin(), u2.getName()), tuple(u3.getLogin(), u3.getName()));
+  }
+
+  @Test
+  public void scrollByLogins_scrolls_by_pages_of_1000_logins() {
+    List<String> logins = new ArrayList<>();
+    for (int i = 0; i < DatabaseUtils.PARTITION_SIZE_FOR_ORACLE + 10; i++) {
+      logins.add(insertUser(true).getLogin());
+    }
+
+    List<UserDto> result = new ArrayList<>();
+    underTest.scrollByLogins(db.getSession(), logins, result::add);
+
+    assertThat(result)
+      .extracting(UserDto::getLogin)
+      .containsExactlyInAnyOrder(logins.toArray(new String[0]));
+  }
+
+  @Test
+  public void scrollAll() {
+    UserDto u1 = insertUser(true);
+    UserDto u2 = insertUser(false);
+
+    List<UserDto> result = new ArrayList<>();
+    underTest.scrollAll(db.getSession(), result::add);
+
+    assertThat(result).extracting(UserDto::getLogin, UserDto::getName)
+      .containsExactlyInAnyOrder(tuple(u1.getLogin(), u1.getName()), tuple(u2.getLogin(), u2.getName()));
+  }
+
   private void commit(Runnable runnable) {
     runnable.run();
     session.commit();
   }
 
-  private UserDto newActiveUser() {
+  private UserDto insertActiveUser() {
     return insertUser(true);
   }
 

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v62/UpdateQualityGateConditionsOnCoverage.java

@@ -39,9 +39,9 @@ import org.sonar.server.platform.db.migration.step.Select;
 import org.sonar.server.platform.db.migration.step.Upsert;
 
 import static java.util.Objects.requireNonNull;
+import static org.apache.commons.lang.StringUtils.repeat;
 import static org.sonar.core.util.stream.MoreCollectors.index;
 import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex;
-import static org.sonar.db.DatabaseUtils.repeatCondition;
 
 /**
  * Migrate every quality gates that have conditions on related coverage metrics.
@@ -144,7 +144,7 @@ public class UpdateQualityGateConditionsOnCoverage extends DataChange {
       try {
         Select select = context.prepareSelect("select qgc.id, qgc.metric_id " +
           "from quality_gate_conditions qgc " +
-          "where qgc.qgate_id=? and qgc.metric_id in (" + repeatCondition("?", metricIds.size(), ",") + ")")
+          "where qgc.qgate_id=? and qgc.metric_id in (" + repeat("?", " , ", metricIds.size()) + ")")
           .setLong(1, qualityGateId);
         for (int i = 0; i < metricIds.size(); i++) {
           select.setLong(i + 2, metricIds.get(i));
@@ -192,7 +192,7 @@ public class UpdateQualityGateConditionsOnCoverage extends DataChange {
     metricKeys.addAll(COVERAGE_METRIC_KEYS.stream().map(metricKey -> IT_PREFIX + metricKey).collect(Collectors.toList()));
     metricKeys.addAll(COVERAGE_METRIC_KEYS.stream().map(metricKey -> OVERALL_PREFIX + metricKey).collect(Collectors.toList()));
     metricKeys.addAll(metricKeys.stream().map(metricKey -> NEW_PREFIX + metricKey).collect(Collectors.toList()));
-    Select select = context.prepareSelect("select id, name from metrics where name in (" + repeatCondition("?", metricKeys.size(), ",") + ")");
+    Select select = context.prepareSelect("select id, name from metrics where name in (" + repeat("?", ",", metricKeys.size()) + ")");
     for (int i = 0; i < metricKeys.size(); i++) {
       select.setString(i + 1, metricKeys.get(i));
     }

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAt.java

@@ -0,0 +1,48 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.platform.db.migration.version.v65;
+
+import java.sql.SQLException;
+import org.sonar.db.Database;
+import org.sonar.server.platform.db.migration.def.BigIntegerColumnDef;
+import org.sonar.server.platform.db.migration.sql.CreateIndexBuilder;
+import org.sonar.server.platform.db.migration.step.DdlChange;
+
+public class AddIndexOnEsQueueCreatedAt extends DdlChange {
+
+  public AddIndexOnEsQueueCreatedAt(Database db) {
+    super(db);
+  }
+
+  @Override
+  public void execute(Context context) throws SQLException {
+    BigIntegerColumnDef column = BigIntegerColumnDef.newBigIntegerColumnDefBuilder()
+      .setColumnName("created_at")
+      .setIsNullable(false)
+      .build();
+
+    context.execute(new CreateIndexBuilder(getDialect())
+      .setName("es_queue_created_at")
+      .setTable("es_queue")
+      .addColumn(column)
+      .build());
+  }
+}

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java

@@ -0,0 +1,63 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.platform.db.migration.version.v65;
+
+import java.sql.SQLException;
+import org.sonar.db.Database;
+import org.sonar.server.platform.db.migration.def.BigIntegerColumnDef;
+import org.sonar.server.platform.db.migration.def.VarcharColumnDef;
+import org.sonar.server.platform.db.migration.sql.CreateTableBuilder;
+import org.sonar.server.platform.db.migration.step.DdlChange;
+
+public class CreateEsQueueTable extends DdlChange {
+
+  private static final String TABLE_NAME = "es_queue";
+
+  public CreateEsQueueTable(Database db) {
+    super(db);
+  }
+
+  @Override
+  public void execute(Context context) throws SQLException {
+    context.execute(new CreateTableBuilder(getDialect(), TABLE_NAME)
+      .addPkColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
+        .setColumnName("uuid")
+        .setIsNullable(false)
+        .setLimit(40)
+        .build())
+      .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
+        .setColumnName("doc_type")
+        .setIsNullable(false)
+        .setLimit(40)
+        .build())
+      .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
+        .setColumnName("doc_uuid")
+        .setIsNullable(false)
+        .setLimit(255)
+        .build())
+      .addColumn(BigIntegerColumnDef.newBigIntegerColumnDefBuilder()
+        .setColumnName("created_at")
+        .setIsNullable(false)
+        .build())
+      .build()
+    );
+  }
+}

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65.java

@@ -60,6 +60,8 @@ public class DbVersion65 implements DbVersion {
       .add(1730, "Add USERS.ONBOARDED", AddUsersOnboarded.class)
       .add(1731, "Populate USERS.ONBOARDED", PopulateUsersOnboarded.class)
       .add(1732, "Make USERS.ONBOARDED not nullable", MakeUsersOnboardedNotNullable.class)
+      .add(1733, "Create table es_queue", CreateEsQueueTable.class)
+      .add(1734, "Add index on es_queue.created_at", AddIndexOnEsQueueCreatedAt.class)
     ;
   }
 }

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest.java

@@ -0,0 +1,41 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v65;
+
+import java.sql.SQLException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.db.CoreDbTester;
+import org.sonar.server.platform.db.migration.step.DdlChange;
+
+public class AddIndexOnEsQueueCreatedAtTest {
+
+  @Rule
+  public CoreDbTester db = CoreDbTester.createForSchema(AddIndexOnEsQueueCreatedAtTest.class, "initial.sql");
+
+  private DdlChange underTest = new AddIndexOnEsQueueCreatedAt(db.database());
+
+  @Test
+  public void add_index() throws SQLException {
+    underTest.execute();
+
+    db.assertIndex("es_queue", "es_queue_created_at", "created_at");
+  }
+}

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java

@@ -0,0 +1,54 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.platform.db.migration.version.v65;
+
+import java.sql.SQLException;
+import java.sql.Types;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonar.db.CoreDbTester;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CreateEsQueueTableTest {
+
+  private static final String TABLE = "es_queue";
+
+  @Rule
+  public final CoreDbTester db = CoreDbTester.createForSchema(CreateEsQueueTableTest.class, "empty.sql");
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  private CreateEsQueueTable underTest = new CreateEsQueueTable(db.database());
+
+  @Test
+  public void creates_table_on_empty_db() throws SQLException {
+    underTest.execute();
+
+    assertThat(db.countRowsOfTable(TABLE)).isEqualTo(0);
+
+    db.assertColumnDefinition(TABLE, "uuid", Types.VARCHAR, 40, false);
+    db.assertColumnDefinition(TABLE, "doc_type", Types.VARCHAR, 40, false);
+    db.assertColumnDefinition(TABLE, "doc_uuid", Types.VARCHAR, 255, false);
+    db.assertColumnDefinition(TABLE, "created_at", Types.BIGINT, null, false);
+  }
+}

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65Test.java

@@ -35,6 +35,6 @@ public class DbVersion65Test {
 
   @Test
   public void verify_migration_count() {
-    verifyMigrationCount(underTest, 33);
+    verifyMigrationCount(underTest, 35);
   }
 }

server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest/initial.sql

@@ -0,0 +1,7 @@
+CREATE TABLE "ES_QUEUE" (
+  "UUID" VARCHAR(40) NOT NULL PRIMARY KEY,
+  "DOC_TYPE" VARCHAR(40) NOT NULL,
+  "DOC_UUID" VARCHAR(255) NOT NULL,
+  "CREATED_AT" BIGINT NOT NULL
+);
+CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID");

server/sonar-server/src/main/java/org/sonar/server/authentication/UserIdentityAuthenticator.java

@@ -86,21 +86,21 @@ public class UserIdentityAuthenticator {
     }
   }
 
-  private UserDto registerNewUser(DbSession dbSession, UserIdentity user, IdentityProvider provider, AuthenticationEvent.Source source) {
+  private UserDto registerNewUser(DbSession dbSession, UserIdentity identity, IdentityProvider provider, AuthenticationEvent.Source source) {
     if (!provider.allowsUsersToSignUp()) {
       throw AuthenticationException.newBuilder()
         .setSource(source)
-        .setLogin(user.getLogin())
+        .setLogin(identity.getLogin())
         .setMessage(format("User signup disabled for provider '%s'", provider.getKey()))
         .setPublicMessage(format("'%s' users are not allowed to sign up", provider.getKey()))
         .build();
     }
 
-    String email = user.getEmail();
+    String email = identity.getEmail();
     if (email != null && dbClient.userDao().doesEmailExist(dbSession, email)) {
       throw AuthenticationException.newBuilder()
         .setSource(source)
-        .setLogin(user.getLogin())
+        .setLogin(identity.getLogin())
         .setMessage(format("Email '%s' is already used", email))
         .setPublicMessage(format(
           "You can't sign up because email '%s' is already used by an existing user. This means that you probably already registered with another account.",
@@ -108,25 +108,22 @@ public class UserIdentityAuthenticator {
         .build();
     }
 
-    String userLogin = user.getLogin();
-    userUpdater.create(dbSession, NewUser.builder()
+    String userLogin = identity.getLogin();
+    return userUpdater.createAndCommit(dbSession, NewUser.builder()
       .setLogin(userLogin)
-      .setEmail(user.getEmail())
-      .setName(user.getName())
-      .setExternalIdentity(new ExternalIdentity(provider.getKey(), user.getProviderLogin()))
-      .build());
-    UserDto newUser = dbClient.userDao().selectOrFailByLogin(dbSession, userLogin);
-    syncGroups(dbSession, user, newUser);
-    return newUser;
+      .setEmail(identity.getEmail())
+      .setName(identity.getName())
+      .setExternalIdentity(new ExternalIdentity(provider.getKey(), identity.getProviderLogin()))
+      .build(), u -> syncGroups(dbSession, identity, u));
   }
 
-  private void registerExistingUser(DbSession dbSession, UserDto userDto, UserIdentity user, IdentityProvider provider) {
-    userUpdater.update(dbSession, UpdateUser.create(userDto.getLogin())
-      .setEmail(user.getEmail())
-      .setName(user.getName())
-      .setExternalIdentity(new ExternalIdentity(provider.getKey(), user.getProviderLogin()))
-      .setPassword(null));
-    syncGroups(dbSession, user, userDto);
+  private void registerExistingUser(DbSession dbSession, UserDto userDto, UserIdentity identity, IdentityProvider provider) {
+    UpdateUser update = UpdateUser.create(userDto.getLogin())
+      .setEmail(identity.getEmail())
+      .setName(identity.getName())
+      .setExternalIdentity(new ExternalIdentity(provider.getKey(), identity.getProviderLogin()))
+      .setPassword(null);
+    userUpdater.updateAndCommit(dbSession, update, u -> syncGroups(dbSession, identity, u));
   }
 
   private void syncGroups(DbSession dbSession, UserIdentity userIdentity, UserDto userDto) {
@@ -149,8 +146,6 @@ public class UserIdentityAuthenticator {
 
     addGroups(dbSession, userDto, groupsToAdd, groupsByName);
     removeGroups(dbSession, userDto, groupsToRemove, groupsByName);
-
-    dbSession.commit();
   }
 
   private void addGroups(DbSession dbSession, UserDto userDto, Collection<String> groupsToAdd, Map<String, GroupDto> groupsByName) {

server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java

@@ -59,7 +59,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     doIndexByProjectUuid(null, Size.LARGE);
   }
 

server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java

@@ -1,91 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.Uninterruptibles;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import org.picocontainer.Startable;
-import org.sonar.api.utils.System2;
-
-public abstract class BaseIndexer implements Startable {
-
-  private final System2 system2;
-  private final ThreadPoolExecutor executor;
-  private final IndexType indexType;
-  protected final EsClient esClient;
-  private final String dateFieldName;
-  private volatile long lastUpdatedAt = -1L;
-
-  protected BaseIndexer(System2 system2, EsClient client, long threadKeepAliveSeconds, IndexType indexType,
-    String dateFieldName) {
-    this.system2 = system2;
-    this.indexType = indexType;
-    this.dateFieldName = dateFieldName;
-    this.esClient = client;
-    this.executor = new ThreadPoolExecutor(0, 1,
-      threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
-  }
-
-  public void index(final IndexerTask task) {
-    final long requestedAt = system2.now();
-    Future submit = executor.submit(() -> {
-      if (lastUpdatedAt == -1L) {
-        lastUpdatedAt = esClient.getMaxFieldValue(indexType, dateFieldName);
-      }
-      if (requestedAt > lastUpdatedAt) {
-        long l = task.index(lastUpdatedAt);
-        // l can be 0 if no documents were indexed
-        lastUpdatedAt = Math.max(l, lastUpdatedAt);
-      }
-    });
-    try {
-      Uninterruptibles.getUninterruptibly(submit);
-    } catch (ExecutionException e) {
-      Throwables.propagate(e);
-    }
-  }
-
-  public void index() {
-    index(this::doIndex);
-  }
-
-  protected abstract long doIndex(long lastUpdatedAt);
-
-  @Override
-  public void start() {
-    // nothing to do at startup
-  }
-
-  @Override
-  public void stop() {
-    executor.shutdown();
-  }
-
-  @FunctionalInterface
-  public interface IndexerTask {
-    long index(long lastUpdatedAt);
-  }
-
-}

server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java

@@ -20,10 +20,15 @@
 package org.sonar.server.es;
 
 import com.google.common.annotations.VisibleForTesting;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import javax.annotation.Nullable;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
@@ -43,12 +48,15 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.sort.SortOrder;
-import org.picocontainer.Startable;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.core.util.ProgressLogger;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
 
 import static java.lang.String.format;
+import static java.util.stream.Collectors.toList;
 
 /**
  * Helper to bulk requests in an efficient way :
@@ -57,7 +65,7 @@ import static java.lang.String.format;
  *   <li>on large table indexing, replicas and automatic refresh can be temporarily disabled</li>
  * </ul>
  */
-public class BulkIndexer implements Startable {
+public class BulkIndexer {
 
   private static final Logger LOGGER = Loggers.get(BulkIndexer.class);
   private static final ByteSizeValue FLUSH_BYTE_SIZE = new ByteSizeValue(1, ByteSizeUnit.MB);
@@ -69,13 +77,22 @@ public class BulkIndexer implements Startable {
   private final String indexName;
   private final BulkProcessor bulkProcessor;
   private final AtomicLong counter = new AtomicLong(0L);
+  private final AtomicLong successCounter = new AtomicLong(0L);
   private final SizeHandler sizeHandler;
+  private final BulkProcessorListener bulkProcessorListener;
+  @Nullable
+  private DbClient dbClient;
+  @Nullable
+  private DbSession dbSession;
+  private Collection<EsQueueDto> esQueueDtos;
 
   public BulkIndexer(EsClient client, String indexName, Size size) {
+    this.dbClient = null;
     this.client = client;
     this.indexName = indexName;
     this.sizeHandler = size.createHandler(Runtime2.INSTANCE);
-    this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), new BulkProcessorListener())
+    this.bulkProcessorListener = new BulkProcessorListener();
+    this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener)
       .setBackoffPolicy(BackoffPolicy.exponentialBackoff())
       .setBulkSize(FLUSH_BYTE_SIZE)
       .setBulkActions(FLUSH_ACTIONS)
@@ -83,22 +100,36 @@ public class BulkIndexer implements Startable {
       .build();
   }
 
-  @Override
   public void start() {
     sizeHandler.beforeStart(this);
     counter.set(0L);
+    successCounter.set(0L);
   }
 
-  @Override
-  public void stop() {
+  public void start(DbSession dbSession, DbClient dbClient, Collection<EsQueueDto> esQueueDtos) {
+    this.dbClient = dbClient;
+    this.dbSession = dbSession;
+    this.esQueueDtos = esQueueDtos;
+    sizeHandler.beforeStart(this);
+    counter.set(0L);
+    successCounter.set(0L);
+  }
+
+  /**
+   * @return the number of documents successfully indexed
+   */
+  public long stop() {
     try {
-      bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
+      bulkProcessor.awaitClose(1, TimeUnit.MINUTES);
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
-      throw new IllegalStateException("Elasticsearch bulk requests still being executed after 10 minutes", e);
+      throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e);
+    } finally {
+      dbSession = null;
     }
     client.prepareRefresh(indexName).get();
     sizeHandler.afterStop(this);
+    return successCounter.get();
   }
 
   public void add(ActionRequest<?> request) {
@@ -161,7 +192,6 @@ public class BulkIndexer implements Startable {
   }
 
   private final class BulkProcessorListener implements Listener {
-
     @Override
     public void beforeBulk(long executionId, BulkRequest request) {
       // no action required
@@ -174,14 +204,31 @@ public class BulkIndexer implements Startable {
       for (BulkItemResponse item : response.getItems()) {
         if (item.isFailed()) {
           LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage());
+        } else {
+          successCounter.incrementAndGet();
         }
       }
+
+      deleteSuccessfulItems(response);
     }
 
     @Override
     public void afterBulk(long executionId, BulkRequest req, Throwable e) {
       LOGGER.error("Fail to execute bulk index request: " + req, e);
     }
+
+    private void deleteSuccessfulItems(BulkResponse bulkResponse) {
+      if (esQueueDtos != null) {
+        List<EsQueueDto> itemsToDelete = Arrays.stream(bulkResponse.getItems())
+          .filter(b -> !b.isFailed())
+          .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocUuid())).findFirst().orElse(null))
+          .filter(Objects::nonNull)
+          .collect(toList());
+
+        dbClient.esQueueDao().delete(dbSession, itemsToDelete);
+        dbSession.commit();
+      }
+    }
   }
 
   public enum Size {
@@ -293,5 +340,4 @@ public class BulkIndexer implements Startable {
       req.get();
     }
   }
-
 }

server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java

@@ -0,0 +1,161 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.es;
+
+import com.google.common.collect.ListMultimap;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang.math.RandomUtils;
+import org.sonar.api.Startable;
+import org.sonar.api.config.Settings;
+import org.sonar.api.utils.System2;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.api.utils.log.Profiler;
+import org.sonar.core.util.stream.MoreCollectors;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
+import org.sonar.server.user.index.UserIndexer;
+
+import static java.lang.String.format;
+
+public class RecoveryIndexer implements Startable {
+
+  private static final Logger LOGGER = Loggers.get(RecoveryIndexer.class);
+  private static final String LOG_PREFIX = "Elasticsearch recovery - ";
+  private static final String PROPERTY_INITIAL_DELAY = "sonar.search.recovery.initialDelayInMs";
+  private static final String PROPERTY_DELAY = "sonar.search.recovery.delayInMs";
+  private static final String PROPERTY_MIN_AGE = "sonar.search.recovery.minAgeInMs";
+  private static final String PROPERTY_LOOP_LIMIT = "sonar.search.recovery.loopLimit";
+  private static final long DEFAULT_DELAY_IN_MS = 5L * 60 * 1000;
+  private static final long DEFAULT_MIN_AGE_IN_MS = 5L * 60 * 1000;
+  private static final int DEFAULT_LOOP_LIMIT = 10_000;
+  private static final double CIRCUIT_BREAKER_IN_PERCENT = 0.3;
+
+  private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1,
+    new ThreadFactoryBuilder()
+      .setPriority(Thread.MIN_PRIORITY)
+      .setNameFormat("RecoveryIndexer-%d")
+      .build());
+  private final System2 system2;
+  private final Settings settings;
+  private final DbClient dbClient;
+  private final UserIndexer userIndexer;
+  private final long minAgeInMs;
+  private final long loopLimit;
+
+  public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer) {
+    this.system2 = system2;
+    this.settings = settings;
+    this.dbClient = dbClient;
+    this.userIndexer = userIndexer;
+    this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS);
+    this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT);
+  }
+
+  @Override
+  public void start() {
+    long delayInMs = getSetting(PROPERTY_DELAY, DEFAULT_DELAY_IN_MS);
+
+    // in the cluster mode, avoid (but not prevent!) simultaneous executions of recovery
+    // indexers so that a document is not handled multiple times.
+    long initialDelayInMs = getSetting(PROPERTY_INITIAL_DELAY, RandomUtils.nextInt(1 + (int) (delayInMs / 2)));
+
+    executorService.scheduleAtFixedRate(
+      this::recover,
+      initialDelayInMs,
+      delayInMs,
+      TimeUnit.MILLISECONDS);
+  }
+
+  @Override
+  public void stop() {
+    try {
+      executorService.shutdown();
+      executorService.awaitTermination(5, TimeUnit.SECONDS);
+    } catch (InterruptedException e) {
+      LOGGER.error(LOG_PREFIX + "Unable to stop recovery indexer in timely fashion", e);
+      executorService.shutdownNow();
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  void recover() {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      Profiler profiler = Profiler.create(LOGGER).start();
+      long beforeDate = system2.now() - minAgeInMs;
+      long total = 0L;
+      long totalSuccess = 0L;
+
+      Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
+      while (!items.isEmpty()) {
+        total += items.size();
+        long loopSuccess = 0L;
+
+        ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
+        for (Map.Entry<EsQueueDto.Type, Collection<EsQueueDto>> entry : itemsByType.asMap().entrySet()) {
+          loopSuccess += doIndex(dbSession, entry.getKey(), entry.getValue());
+        }
+
+        totalSuccess += loopSuccess;
+        if (1.0d * (items.size() - loopSuccess) / items.size() >= CIRCUIT_BREAKER_IN_PERCENT) {
+          LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", items.size() - loopSuccess, items.size());
+          break;
+        }
+        items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
+      }
+      if (total > 0L) {
+        profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", total, total - totalSuccess));
+      }
+    } catch (Throwable t) {
+      LOGGER.error(LOG_PREFIX + "fail to recover documents", t);
+    }
+  }
+
+  private long doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) {
+    LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type);
+    switch (type) {
+      case USER:
+        return userIndexer.index(dbSession, typeItems);
+      default:
+        LOGGER.error(LOG_PREFIX + "ignore {} documents with unsupported type {}", typeItems.size(), type);
+        return 0;
+    }
+  }
+
+  private static ListMultimap<EsQueueDto.Type, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) {
+    return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType));
+  }
+
+  private long getSetting(String key, long defaultValue) {
+    long val = settings.getLong(key);
+    if (val <= 0) {
+      val = defaultValue;
+    }
+    LOGGER.debug(LOG_PREFIX + "{}={}", key, val);
+    return val;
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/es/StartupIndexer.java

@@ -28,9 +28,9 @@ public interface StartupIndexer {
 
   /**
    * This reindexing method will only be called on startup, and only,
-   * if there is at least one empty types.
+   * if there is at least one uninitialized type.
    */
-  void indexOnStartup(Set<IndexType> emptyIndexTypes);
+  void indexOnStartup(Set<IndexType> uninitializedIndexTypes);
 
   Set<IndexType> getIndexTypes();
 

server/sonar-server/src/main/java/org/sonar/server/es/queue/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.server.es.queue;
+
+import javax.annotation.ParametersAreNonnullByDefault;

server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java

@@ -69,7 +69,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     doIndex(createBulkIndexer(Size.LARGE), (String) null);
   }
 

server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java

@@ -59,7 +59,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     doIndex(createBulkIndexer(Size.LARGE), (String) null);
   }
 

server/sonar-server/src/main/java/org/sonar/server/organization/OrganizationCreationImpl.java

@@ -106,11 +106,10 @@ public class OrganizationCreationImpl implements OrganizationCreation {
       addCurrentUserToGroup(dbSession, ownerGroup, userCreator.getId());
       addCurrentUserToGroup(dbSession, defaultGroup, userCreator.getId());
 
-      dbSession.commit();
       batchDbSession.commit();
 
       // Elasticsearch is updated when DB session is committed
-      userIndexer.index(userCreator.getLogin());
+      userIndexer.commitAndIndex(dbSession, userCreator);
 
       return organization;
     }
@@ -144,11 +143,10 @@ public class OrganizationCreationImpl implements OrganizationCreation {
       insertQualityProfiles(dbSession, batchDbSession, organization);
       addCurrentUserToGroup(dbSession, defaultGroup, newUser.getId());
 
-      dbSession.commit();
       batchDbSession.commit();
 
       // Elasticsearch is updated when DB session is committed
-      userIndexer.index(newUser.getLogin());
+      userIndexer.commitAndIndex(dbSession, newUser);
 
       return Optional.of(organization);
     }

server/sonar-server/src/main/java/org/sonar/server/organization/ws/AddMemberAction.java

@@ -118,8 +118,7 @@ public class AddMemberAction implements OrganizationsWsAction {
       .setUserId(user.getId()));
     dbClient.userGroupDao().insert(dbSession,
       new UserGroupDto().setGroupId(defaultGroupFinder.findDefaultGroup(dbSession, organization.getUuid()).getId()).setUserId(user.getId()));
-    dbSession.commit();
-    userIndexer.index(user.getLogin());
+    userIndexer.commitAndIndex(dbSession, user);
   }
 
   private AddMemberWsResponse buildResponse(UserDto user, int groups) {

server/sonar-server/src/main/java/org/sonar/server/organization/ws/DeleteAction.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.server.organization.ws;
 
+import java.util.Collection;
 import java.util.List;
 import org.sonar.api.server.ws.Request;
 import org.sonar.api.server.ws.Response;
@@ -138,11 +139,10 @@ public class DeleteAction implements OrganizationsWsAction {
   }
 
   private void deleteOrganization(DbSession dbSession, OrganizationDto organization) {
-    List<String> logins = dbClient.organizationMemberDao().selectLoginsByOrganizationUuid(dbSession, organization.getUuid());
+    Collection<String> logins = dbClient.organizationMemberDao().selectLoginsByOrganizationUuid(dbSession, organization.getUuid());
     dbClient.organizationMemberDao().deleteByOrganizationUuid(dbSession, organization.getUuid());
     dbClient.organizationDao().deleteByUuid(dbSession, organization.getUuid());
-    dbSession.commit();
-    userIndexer.index(logins);
+    userIndexer.commitAndIndexByLogins(dbSession, logins);
   }
 
   private static void preventDeletionOfDefaultOrganization(String key, DefaultOrganization defaultOrganization) {

server/sonar-server/src/main/java/org/sonar/server/organization/ws/RemoveMemberAction.java

@@ -103,8 +103,7 @@ public class RemoveMemberAction implements OrganizationsWsAction {
     dbClient.propertiesDao().deleteByOrganizationAndMatchingLogin(dbSession, organizationUuid, user.getLogin(), singletonList(DEFAULT_ISSUE_ASSIGNEE));
 
     dbClient.organizationMemberDao().delete(dbSession, organizationUuid, userId);
-    dbSession.commit();
-    userIndexer.index(user.getLogin());
+    userIndexer.commitAndIndex(dbSession, user);
   }
 
   private void ensureLastAdminIsNotRemoved(DbSession dbSession, OrganizationDto organizationDto, UserDto user) {

server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java

@@ -80,9 +80,9 @@ public class PermissionIndexer implements ProjectIndexer, StartupIndexer {
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     List<Dto> authorizations = getAllAuthorizations();
-    Stream<AuthorizationScope> scopes = getScopes(emptyIndexTypes);
+    Stream<AuthorizationScope> scopes = getScopes(uninitializedIndexTypes);
     index(authorizations, scopes, Size.LARGE);
   }
 

server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java

@@ -32,8 +32,8 @@ import org.apache.commons.lang.StringUtils;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 
+import static org.apache.commons.lang.StringUtils.repeat;
 import static org.sonar.db.DatabaseUtils.executeLargeInputs;
-import static org.sonar.db.DatabaseUtils.repeatCondition;
 
 /**
  * No streaming because of union of joins -> no need to use ResultSetIterator
@@ -200,7 +200,7 @@ public class PermissionIndexerDao {
     if (projectUuids.isEmpty()) {
       sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", "");
     } else {
-      sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", " AND (" + repeatCondition("projects.uuid = ?", projectUuids.size(), "OR") + ")");
+      sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", " AND (" + repeat("projects.uuid = ?", " or ", projectUuids.size()) + ")");
     }
     PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
     int index = 1;

server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java

@@ -53,6 +53,7 @@ import org.sonar.server.duplication.ws.ShowResponseBuilder;
 import org.sonar.server.email.ws.EmailsWsModule;
 import org.sonar.server.es.IndexCreator;
 import org.sonar.server.es.IndexDefinitions;
+import org.sonar.server.es.RecoveryIndexer;
 import org.sonar.server.event.NewAlerts;
 import org.sonar.server.favorite.FavoriteModule;
 import org.sonar.server.issue.AddTagsAction;
@@ -520,7 +521,9 @@ public class PlatformLevel4 extends PlatformLevel {
       WebhooksWsModule.class,
 
       // Http Request ID
-      HttpRequestIdModule.class);
+      HttpRequestIdModule.class,
+
+      RecoveryIndexer.class);
     addAll(level4AddedComponents);
   }
 

server/sonar-server/src/main/java/org/sonar/server/property/InternalProperties.java

@@ -34,6 +34,8 @@ public interface InternalProperties {
 
   String ORGANIZATION_ENABLED = "organization.enabled";
 
+  String ES_INDEX_INITIALIZING_PREFIX = "es.initializing.";
+
   /**
    * Read the value of the specified property.
    *

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java

@@ -58,7 +58,7 @@ public class ActiveRuleIndexer implements StartupIndexer {
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     try (DbSession dbSession = dbClient.openSession(false)) {
       ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForAll(dbSession);
       scrollDbAndIndex(dbCursor, Size.LARGE);

server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java

@@ -77,7 +77,7 @@ public class TestIndexer implements ProjectIndexer, StartupIndexer {
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     doIndex(null, Size.LARGE);
   }
 

server/sonar-server/src/main/java/org/sonar/server/user/UserUpdater.java

@@ -22,16 +22,17 @@ package org.sonar.server.user;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
 import java.security.SecureRandom;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Random;
+import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.sonar.api.config.Configuration;
 import org.sonar.api.platform.NewUserHandler;
 import org.sonar.api.server.ServerSide;
-import org.sonar.api.utils.System2;
 import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
@@ -73,19 +74,17 @@ public class UserUpdater {
   private final NewUserNotifier newUserNotifier;
   private final DbClient dbClient;
   private final UserIndexer userIndexer;
-  private final System2 system2;
   private final OrganizationFlags organizationFlags;
   private final DefaultOrganizationProvider defaultOrganizationProvider;
   private final OrganizationCreation organizationCreation;
   private final DefaultGroupFinder defaultGroupFinder;
   private final Configuration config;
 
-  public UserUpdater(NewUserNotifier newUserNotifier, DbClient dbClient, UserIndexer userIndexer, System2 system2, OrganizationFlags organizationFlags,
+  public UserUpdater(NewUserNotifier newUserNotifier, DbClient dbClient, UserIndexer userIndexer, OrganizationFlags organizationFlags,
     DefaultOrganizationProvider defaultOrganizationProvider, OrganizationCreation organizationCreation, DefaultGroupFinder defaultGroupFinder, Configuration config) {
     this.newUserNotifier = newUserNotifier;
     this.dbClient = dbClient;
     this.userIndexer = userIndexer;
-    this.system2 = system2;
     this.organizationFlags = organizationFlags;
     this.defaultOrganizationProvider = defaultOrganizationProvider;
     this.organizationCreation = organizationCreation;
@@ -93,14 +92,17 @@ public class UserUpdater {
     this.config = config;
   }
 
-  public UserDto create(DbSession dbSession, NewUser newUser) {
+  public UserDto createAndCommit(DbSession dbSession, NewUser newUser, Consumer<UserDto> beforeCommit) {
     String login = newUser.login();
     UserDto userDto = dbClient.userDao().selectByLogin(dbSession, newUser.login());
     if (userDto == null) {
-      userDto = saveUser(dbSession, createNewUserDto(dbSession, newUser));
+      userDto = saveUser(dbSession, createDto(dbSession, newUser));
     } else {
       reactivateUser(dbSession, userDto, login, newUser);
     }
+    beforeCommit.accept(userDto);
+    userIndexer.commitAndIndex(dbSession, userDto);
+
     notifyNewUser(userDto.getLogin(), userDto.getName(), newUser.email());
     return userDto;
   }
@@ -120,26 +122,31 @@ public class UserUpdater {
     // Hack to allow to change the password of the user
     existingUser.setLocal(true);
     setOnboarded(existingUser);
-    updateUserDto(dbSession, updateUser, existingUser);
+    updateDto(dbSession, updateUser, existingUser);
     updateUser(dbSession, existingUser);
     addUserToDefaultOrganizationAndDefaultGroup(dbSession, existingUser);
-    dbSession.commit();
   }
 
-  public void update(DbSession dbSession, UpdateUser updateUser) {
-    UserDto user = dbClient.userDao().selectByLogin(dbSession, updateUser.login());
-    checkFound(user, "User with login '%s' has not been found", updateUser.login());
-    boolean isUserUpdated = updateUserDto(dbSession, updateUser, user);
-    if (!isUserUpdated) {
-      return;
+  public void updateAndCommit(DbSession dbSession, UpdateUser updateUser, Consumer<UserDto> beforeCommit) {
+    UserDto dto = dbClient.userDao().selectByLogin(dbSession, updateUser.login());
+    checkFound(dto, "User with login '%s' has not been found", updateUser.login());
+    boolean isUserUpdated = updateDto(dbSession, updateUser, dto);
+    if (isUserUpdated) {
+      // at least one change. Database must be updated and Elasticsearch re-indexed
+      updateUser(dbSession, dto);
+      beforeCommit.accept(dto);
+      userIndexer.commitAndIndex(dbSession, dto);
+      notifyNewUser(dto.getLogin(), dto.getName(), dto.getEmail());
+    } else {
+      // no changes but still execute the consumer
+      beforeCommit.accept(dto);
+      dbSession.commit();
     }
-    updateUser(dbSession, user);
-    notifyNewUser(user.getLogin(), user.getName(), user.getEmail());
   }
 
-  private UserDto createNewUserDto(DbSession dbSession, NewUser newUser) {
+  private UserDto createDto(DbSession dbSession, NewUser newUser) {
     UserDto userDto = new UserDto();
-    List<String> messages = newArrayList();
+    List<String> messages = new ArrayList<>();
 
     String login = newUser.login();
     if (validateLoginFormat(login, messages)) {
@@ -158,7 +165,7 @@ public class UserUpdater {
 
     String password = newUser.password();
     if (password != null && validatePasswords(password, messages)) {
-      setEncryptedPassWord(password, userDto);
+      setEncryptedPassword(password, userDto);
     }
 
     List<String> scmAccounts = sanitizeScmAccounts(newUser.scmAccounts());
@@ -173,13 +180,13 @@ public class UserUpdater {
     return userDto;
   }
 
-  private boolean updateUserDto(DbSession dbSession, UpdateUser updateUser, UserDto userDto) {
+  private boolean updateDto(DbSession dbSession, UpdateUser update, UserDto dto) {
     List<String> messages = newArrayList();
-    boolean changed = updateName(updateUser, userDto, messages);
-    changed |= updateEmail(updateUser, userDto, messages);
-    changed |= updateExternalIdentity(updateUser, userDto);
-    changed |= updatePassword(updateUser, userDto, messages);
-    changed |= updateScmAccounts(dbSession, updateUser, userDto, messages);
+    boolean changed = updateName(update, dto, messages);
+    changed |= updateEmail(update, dto, messages);
+    changed |= updateExternalIdentity(update, dto);
+    changed |= updatePassword(update, dto, messages);
+    changed |= updateScmAccounts(dbSession, update, dto, messages);
     checkRequest(messages.isEmpty(), messages);
     return changed;
   }
@@ -216,7 +223,7 @@ public class UserUpdater {
   private static boolean updatePassword(UpdateUser updateUser, UserDto userDto, List<String> messages) {
     String password = updateUser.password();
     if (!updateUser.isExternalIdentityChanged() && updateUser.isPasswordChanged() && validatePasswords(password, messages) && checkPasswordChangeAllowed(userDto, messages)) {
-      setEncryptedPassWord(password, userDto);
+      setEncryptedPassword(password, userDto);
       return true;
     }
     return false;
@@ -355,25 +362,19 @@ public class UserUpdater {
   }
 
   private UserDto saveUser(DbSession dbSession, UserDto userDto) {
-    long now = system2.now();
-    userDto.setActive(true).setCreatedAt(now).setUpdatedAt(now);
+    userDto.setActive(true);
     UserDto res = dbClient.userDao().insert(dbSession, userDto);
     addUserToDefaultOrganizationAndDefaultGroup(dbSession, userDto);
     organizationCreation.createForUser(dbSession, userDto);
-    dbSession.commit();
-    userIndexer.index(userDto.getLogin());
     return res;
   }
 
-  private void updateUser(DbSession dbSession, UserDto userDto) {
-    long now = system2.now();
-    userDto.setActive(true).setUpdatedAt(now);
-    dbClient.userDao().update(dbSession, userDto);
-    dbSession.commit();
-    userIndexer.index(userDto.getLogin());
+  private void updateUser(DbSession dbSession, UserDto dto) {
+    dto.setActive(true);
+    dbClient.userDao().update(dbSession, dto);
   }
 
-  private static void setEncryptedPassWord(String password, UserDto userDto) {
+  private static void setEncryptedPassword(String password, UserDto userDto) {
     Random random = new SecureRandom();
     byte[] salt = new byte[32];
     random.nextBytes(salt);
@@ -382,7 +383,7 @@ public class UserUpdater {
     userDto.setCryptedPassword(encryptPassword(password, saltHex));
   }
 
-  private void notifyNewUser(String login, String name, String email) {
+  private void notifyNewUser(String login, String name, @Nullable String email) {
     newUserNotifier.onNewUser(NewUserHandler.Context.builder()
       .setLogin(login)
       .setName(name)

server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java

@@ -20,7 +20,6 @@
 package org.sonar.server.user.index;
 
 import com.google.common.collect.Maps;
-import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
@@ -83,14 +82,6 @@ public class UserDoc extends BaseDoc implements User {
     return getField(FIELD_ORGANIZATION_UUIDS);
   }
 
-  public long createdAt() {
-    return getFieldAsDate(UserIndexDefinition.FIELD_CREATED_AT).getTime();
-  }
-
-  public long updatedAt() {
-    return getFieldAsDate(UserIndexDefinition.FIELD_UPDATED_AT).getTime();
-  }
-
   public UserDoc setLogin(@Nullable String s) {
     setField(UserIndexDefinition.FIELD_LOGIN, s);
     return this;
@@ -120,14 +111,4 @@ public class UserDoc extends BaseDoc implements User {
     setField(FIELD_ORGANIZATION_UUIDS, organizationUuids);
     return this;
   }
-
-  public UserDoc setCreatedAt(long l) {
-    setField(UserIndexDefinition.FIELD_CREATED_AT, new Date(l));
-    return this;
-  }
-
-  public UserDoc setUpdatedAt(long l) {
-    setField(UserIndexDefinition.FIELD_UPDATED_AT, new Date(l));
-    return this;
-  }
 }

server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java

@@ -36,8 +36,6 @@ public class UserIndexDefinition implements IndexDefinition {
   public static final String FIELD_LOGIN = "login";
   public static final String FIELD_NAME = "name";
   public static final String FIELD_EMAIL = "email";
-  public static final String FIELD_CREATED_AT = "createdAt";
-  public static final String FIELD_UPDATED_AT = "updatedAt";
   public static final String FIELD_ACTIVE = "active";
   public static final String FIELD_SCM_ACCOUNTS = "scmAccounts";
   public static final String FIELD_ORGANIZATION_UUIDS = "organizationUuids";
@@ -59,8 +57,6 @@ public class UserIndexDefinition implements IndexDefinition {
     mapping.stringFieldBuilder(FIELD_LOGIN).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build();
     mapping.stringFieldBuilder(FIELD_NAME).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build();
     mapping.stringFieldBuilder(FIELD_EMAIL).addSubFields(USER_SEARCH_GRAMS_ANALYZER, SORTABLE_ANALYZER).build();
-    mapping.createDateTimeField(FIELD_CREATED_AT);
-    mapping.createDateTimeField(FIELD_UPDATED_AT);
     mapping.createBooleanField(FIELD_ACTIVE);
     mapping.stringFieldBuilder(FIELD_SCM_ACCOUNTS).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
     mapping.stringFieldBuilder(FIELD_ORGANIZATION_UUIDS).disableNorms().build();

server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java

@@ -19,14 +19,20 @@
  */
 package org.sonar.server.user.index;
 
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Collections2;
 import com.google.common.collect.ImmutableSet;
-import java.util.Iterator;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Maps;
+import java.util.Collection;
 import java.util.List;
 import java.util.Set;
-import javax.annotation.Nullable;
 import org.elasticsearch.action.index.IndexRequest;
+import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
+import org.sonar.db.user.UserDto;
 import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.BulkIndexer.Size;
 import org.sonar.server.es.EsClient;
@@ -35,7 +41,7 @@ import org.sonar.server.es.StartupIndexer;
 
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
-import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput;
+import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
 import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
 
 public class UserIndexer implements StartupIndexer {
@@ -54,56 +60,98 @@ public class UserIndexer implements StartupIndexer {
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
-    doIndex(newBulkIndexer(Size.LARGE), null);
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
+      dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByLogin::put);
+
+      BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE);
+      bulkIndexer.start();
+      dbClient.userDao().scrollAll(dbSession,
+        // only index requests, no deletion requests.
+        // Deactivated users are not deleted but updated.
+        u -> bulkIndexer.add(newIndexRequest(u, organizationUuidsByLogin)));
+      bulkIndexer.stop();
+    }
   }
 
-  public void index(String login) {
-    requireNonNull(login);
-    doIndex(newBulkIndexer(Size.REGULAR), singletonList(login));
+  public void commitAndIndex(DbSession dbSession, UserDto user) {
+    commitAndIndexByLogins(dbSession, singletonList(user.getLogin()));
   }
 
-  public void index(List<String> logins) {
-    requireNonNull(logins);
-    if (logins.isEmpty()) {
-      return;
-    }
-
-    doIndex(newBulkIndexer(Size.REGULAR), logins);
+  public void commitAndIndex(DbSession dbSession, Collection<UserDto> users) {
+    commitAndIndexByLogins(dbSession, Collections2.transform(users, UserDto::getLogin));
   }
 
-  private void doIndex(BulkIndexer bulk, @Nullable List<String> logins) {
-    try (DbSession dbSession = dbClient.openSession(false)) {
-      if (logins == null) {
-        processLogins(bulk, dbSession, null);
-      } else {
-        executeLargeInputsWithoutOutput(logins, l -> processLogins(bulk, dbSession, l));
-      }
-    }
+  public void commitAndIndexByLogins(DbSession dbSession, Collection<String> logins) {
+    List<EsQueueDto> items = logins.stream()
+      .map(l -> EsQueueDto.create(EsQueueDto.Type.USER, l))
+      .collect(MoreCollectors.toArrayList());
+
+    dbClient.esQueueDao().insert(dbSession, items);
+    dbSession.commit();
+    postCommit(dbSession, logins, items);
   }
 
-  private void processLogins(BulkIndexer bulk, DbSession dbSession, @Nullable List<String> logins) {
-    try (UserResultSetIterator rowIt = UserResultSetIterator.create(dbClient, dbSession, logins)) {
-      processResultSet(bulk, rowIt);
-    }
+  /**
+   * Entry point for Byteman tests. See directory tests/resilience.
+   * The parameter "logins" is used only by the Byteman script.
+   */
+  private void postCommit(DbSession dbSession, Collection<String> logins, Collection<EsQueueDto> items) {
+    index(dbSession, items);
   }
 
-  private static void processResultSet(BulkIndexer bulk, Iterator<UserDoc> users) {
-    bulk.start();
-    while (users.hasNext()) {
-      UserDoc user = users.next();
-      bulk.add(newIndexRequest(user));
+  /**
+   * @return the number of items that have been successfully indexed
+   */
+  public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+    if (items.isEmpty()) {
+      return 0L;
     }
-    bulk.stop();
+    Set<String> logins = items
+      .stream()
+      .filter(i -> {
+        requireNonNull(i.getDocUuid(), () -> "BUG - " + i + " has not been persisted before indexing");
+        return true;
+      })
+      .map(EsQueueDto::getDocUuid)
+      .collect(toHashSet(items.size()));
+
+    ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
+    dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put);
+
+    BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR);
+    bulkIndexer.start(dbSession, dbClient, items);
+    dbClient.userDao().scrollByLogins(dbSession, logins,
+      // only index requests, no deletion requests.
+      // Deactivated users are not deleted but updated.
+      u -> {
+        logins.remove(u.getLogin());
+        bulkIndexer.add(newIndexRequest(u, organizationUuidsByLogin));
+      });
+
+    // the remaining logins reference rows that don't exist in db. They must
+    // be deleted from index.
+    logins.forEach(l -> bulkIndexer.addDeletion(UserIndexDefinition.INDEX_TYPE_USER, l));
+    return bulkIndexer.stop();
   }
 
   private BulkIndexer newBulkIndexer(Size bulkSize) {
     return new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex(), bulkSize);
   }
 
-  private static IndexRequest newIndexRequest(UserDoc user) {
-    return new IndexRequest(UserIndexDefinition.INDEX_TYPE_USER.getIndex(), UserIndexDefinition.INDEX_TYPE_USER.getType(), user.login())
-      .source(user.getFields());
+  private static IndexRequest newIndexRequest(UserDto user, ListMultimap<String, String> organizationUuidsByLogins) {
+    UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(8));
+    // all the keys must be present, even if value is null
+    doc.setLogin(user.getLogin());
+    doc.setName(user.getName());
+    doc.setEmail(user.getEmail());
+    doc.setActive(user.isActive());
+    doc.setScmAccounts(UserDto.decodeScmAccounts(user.getScmAccounts()));
+    doc.setOrganizationUuids(organizationUuidsByLogins.get(user.getLogin()));
+
+    return new IndexRequest(UserIndexDefinition.INDEX_TYPE_USER.getIndex(), UserIndexDefinition.INDEX_TYPE_USER.getType())
+      .id(doc.getId())
+      .source(doc.getFields());
   }
-
 }

server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java

@@ -19,22 +19,25 @@
  */
 package org.sonar.server.user.index;
 
-import com.google.common.base.Joiner;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Maps;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Collection;
 import java.util.List;
-import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.ResultSetIterator;
+import org.sonar.db.es.EsQueueDto;
 import org.sonar.db.user.UserDto;
 
+import static org.apache.commons.lang.StringUtils.repeat;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+
 /**
  * Scrolls over table USERS and reads documents to populate the user index
  */
@@ -52,8 +55,6 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> {
   };
 
   private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from users u ";
-  private static final String LOGIN_FILTER = "u.login=?";
-  private static final Joiner OR_JOINER = Joiner.on(" or ");
 
   private final ListMultimap<String, String> organizationUuidsByLogins;
 
@@ -62,16 +63,25 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> {
     this.organizationUuidsByLogins = organizationUuidsByLogins;
   }
 
-  static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable List<String> logins) {
+  static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable Collection<EsQueueDto> esQueueDtos) {
     try {
-      String sql = createSql(logins);
+      String sql = SQL_ALL;
+      List<String> logins = null;
+      if (esQueueDtos != null) {
+        logins = esQueueDtos.stream()
+          .filter(i -> i.getDocType() == EsQueueDto.Type.USER)
+          .map(EsQueueDto::getDocUuid).collect(toArrayList());
+        sql += "where (" + repeat("u.login=?", " or ", logins.size()) + ")";
+      }
+
       PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
       setParameters(stmt, logins);
 
       ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
-      if (logins == null) {
+      if (esQueueDtos == null) {
         dbClient.organizationMemberDao().selectAllForUserIndexing(session, organizationUuidsByLogin::put);
       } else {
+
         dbClient.organizationMemberDao().selectForUserIndexing(session, logins, organizationUuidsByLogin::put);
       }
 
@@ -81,35 +91,21 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> {
     }
   }
 
-  private static String createSql(@Nullable List<String> logins) {
-    if (logins == null) {
-      return SQL_ALL;
-    }
-
-    List<String> sqlLogins = logins.stream()
-      .map(l -> LOGIN_FILTER)
-      .collect(Collectors.toList());
-
-    String sql = SQL_ALL;
-    sql += " WHERE ";
-    sql += "(" + OR_JOINER.join(sqlLogins) + ")";
-
-    return sql;
-  }
-
-  private static void setParameters(PreparedStatement stmt, @Nullable List<String> logins) throws SQLException {
+  private static void setParameters(PreparedStatement stmt, @Nullable Collection<String> logins) throws SQLException {
     if (logins == null) {
       return;
     }
 
-    for (int i = 0; i < logins.size(); i++) {
-      stmt.setString(i + 1, logins.get(i));
+    int paramIndex = 1;
+    for (String login : logins) {
+      stmt.setString(paramIndex, login);
+      paramIndex++;
     }
   }
 
   @Override
   protected UserDoc read(ResultSet rs) throws SQLException {
-    UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(8));
+    UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(6));
 
     String login = rs.getString(1);
 
@@ -119,10 +115,7 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> {
     doc.setEmail(rs.getString(3));
     doc.setActive(rs.getBoolean(4));
     doc.setScmAccounts(UserDto.decodeScmAccounts(rs.getString(5)));
-    doc.setCreatedAt(rs.getLong(6));
-    doc.setUpdatedAt(rs.getLong(7));
     doc.setOrganizationUuids(organizationUuidsByLogins.get(login));
     return doc;
   }
-
 }

server/sonar-server/src/main/java/org/sonar/server/user/ws/ChangePasswordAction.java

@@ -90,7 +90,7 @@ public class ChangePasswordAction implements UsersWsAction {
       String password = request.mandatoryParam(PARAM_PASSWORD);
       UpdateUser updateUser = UpdateUser.create(login).setPassword(password);
 
-      userUpdater.update(dbSession, updateUser);
+      userUpdater.updateAndCommit(dbSession, updateUser, u -> {});
     }
     response.noContent();
   }

server/sonar-server/src/main/java/org/sonar/server/user/ws/CreateAction.java

@@ -127,9 +127,8 @@ public class CreateAction implements UsersWsAction {
       if (!request.isLocal()) {
         newUser.setExternalIdentity(new ExternalIdentity(SQ_AUTHORITY, request.getLogin()));
       }
-      UserDto userDto = userUpdater.create(dbSession, newUser.build());
-      dbSession.commit();
-      return buildResponse(userDto);
+      UserDto createdUser = userUpdater.createAndCommit(dbSession, newUser.build(), u -> {});
+      return buildResponse(createdUser);
     }
   }
 

server/sonar-server/src/main/java/org/sonar/server/user/ws/DeactivateAction.java

@@ -101,11 +101,10 @@ public class DeactivateAction implements UsersWsAction {
       dbClient.userPermissionDao().deleteByUserId(dbSession, userId);
       dbClient.permissionTemplateDao().deleteUserPermissionsByUserId(dbSession, userId);
       dbClient.organizationMemberDao().deleteByUserId(dbSession, userId);
-      dbClient.userDao().deactivateUserById(dbSession, userId);
-      dbSession.commit();
+      dbClient.userDao().deactivateUser(dbSession, user);
+      userIndexer.commitAndIndex(dbSession, user);
     }
 
-    userIndexer.index(login);
     writeResponse(response, login);
   }
 

server/sonar-server/src/main/java/org/sonar/server/user/ws/SkipOnboardingTutorialAction.java

@@ -35,12 +35,10 @@ public class SkipOnboardingTutorialAction implements UsersWsAction {
 
   private final UserSession userSession;
   private final DbClient dbClient;
-  private final System2 system2;
 
-  public SkipOnboardingTutorialAction(UserSession userSession, DbClient dbClient, System2 system2) {
+  public SkipOnboardingTutorialAction(UserSession userSession, DbClient dbClient) {
     this.userSession = userSession;
     this.dbClient = dbClient;
-    this.system2 = system2;
   }
 
   @Override
@@ -63,7 +61,8 @@ public class SkipOnboardingTutorialAction implements UsersWsAction {
       checkState(userDto != null, "User login '%s' cannot be found", userLogin);
       if (!userDto.isOnboarded()) {
         userDto.setOnboarded(true);
-        userDto.setUpdatedAt(system2.now());
+        // no need to update Elasticsearch, the field onBoarded
+        // is not indexed
         dbClient.userDao().update(dbSession, userDto);
         dbSession.commit();
       }

server/sonar-server/src/main/java/org/sonar/server/user/ws/UpdateAction.java

@@ -118,7 +118,7 @@ public class UpdateAction implements UsersWsAction {
     if (!request.getScmAccounts().isEmpty()) {
       updateUser.setScmAccounts(request.getScmAccounts());
     }
-    userUpdater.update(dbSession, updateUser);
+    userUpdater.updateAndCommit(dbSession, updateUser, u -> {});
   }
 
   private void writeUser(DbSession dbSession, Response response, String login) {

server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java

@@ -53,7 +53,7 @@ public class ViewIndexer implements StartupIndexer {
   }
 
   @Override
-  public void indexOnStartup(Set<IndexType> emptyIndexTypes) {
+  public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     try (DbSession dbSession = dbClient.openSession(false)) {
       Map<String, String> viewAndProjectViewUuidMap = newHashMap();
       for (UuidWithProjectUuidDto uuidWithProjectUuidDto : dbClient.componentDao().selectAllViewsAndSubViews(dbSession)) {

server/sonar-server/src/test/java/org/sonar/server/authentication/SsoAuthenticatorTest.java

@@ -40,12 +40,14 @@ import org.sonar.db.user.GroupDto;
 import org.sonar.db.user.UserDto;
 import org.sonar.server.authentication.event.AuthenticationEvent;
 import org.sonar.server.authentication.event.AuthenticationEvent.Source;
+import org.sonar.server.es.EsTester;
 import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.organization.OrganizationCreation;
 import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.organization.TestOrganizationFlags;
 import org.sonar.server.user.NewUserNotifier;
 import org.sonar.server.user.UserUpdater;
+import org.sonar.server.user.index.UserIndexDefinition;
 import org.sonar.server.user.index.UserIndexer;
 import org.sonar.server.usergroups.DefaultGroupFinder;
 
@@ -65,11 +67,14 @@ import static org.sonar.server.authentication.event.AuthenticationExceptionMatch
 
 public class SsoAuthenticatorTest {
 
+  private MapSettings settings = new MapSettings();
+
   @Rule
   public ExpectedException expectedException = none();
-
   @Rule
   public DbTester db = DbTester.create(new AlwaysIncreasingSystem2());
+  @Rule
+  public EsTester es = new EsTester(new UserIndexDefinition(settings.asConfig()));
 
   private static final String DEFAULT_LOGIN = "john";
   private static final String DEFAULT_NAME = "John";
@@ -93,14 +98,14 @@ public class SsoAuthenticatorTest {
   private GroupDto sonarUsers;
 
   private System2 system2 = mock(System2.class);
-  private MapSettings settings = new MapSettings();
   private OrganizationCreation organizationCreation = mock(OrganizationCreation.class);
   private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
   private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone();
 
+  private UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client());
   private UserIdentityAuthenticator userIdentityAuthenticator = new UserIdentityAuthenticator(
     db.getDbClient(),
-    new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), mock(UserIndexer.class), System2.INSTANCE, organizationFlags, defaultOrganizationProvider, organizationCreation,
+    new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), userIndexer, organizationFlags, defaultOrganizationProvider, organizationCreation,
       new DefaultGroupFinder(db.getDbClient()), settings.asConfig()),
     defaultOrganizationProvider, organizationFlags, new DefaultGroupFinder(db.getDbClient()));
 

server/sonar-server/src/test/java/org/sonar/server/authentication/UserIdentityAuthenticatorTest.java

@@ -35,12 +35,14 @@ import org.sonar.db.user.GroupDto;
 import org.sonar.db.user.UserDto;
 import org.sonar.server.authentication.event.AuthenticationEvent.Method;
 import org.sonar.server.authentication.event.AuthenticationEvent.Source;
+import org.sonar.server.es.EsTester;
 import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.organization.OrganizationCreation;
 import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.organization.TestOrganizationFlags;
 import org.sonar.server.user.NewUserNotifier;
 import org.sonar.server.user.UserUpdater;
+import org.sonar.server.user.index.UserIndexDefinition;
 import org.sonar.server.user.index.UserIndexer;
 import org.sonar.server.usergroups.DefaultGroupFinder;
 
@@ -69,32 +71,33 @@ public class UserIdentityAuthenticatorTest {
     .setEnabled(true)
     .setAllowsUsersToSignUp(true);
 
+  private MapSettings settings = new MapSettings();
+
   @Rule
   public ExpectedException thrown = ExpectedException.none();
-
   @Rule
   public DbTester db = DbTester.create(new AlwaysIncreasingSystem2());
-
+  @Rule
+  public EsTester es = new EsTester(new UserIndexDefinition(settings.asConfig()));
+  private UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client());
   private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
   private OrganizationCreation organizationCreation = mock(OrganizationCreation.class);
   private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone();
-  private MapSettings settings = new MapSettings();
-
   private UserUpdater userUpdater = new UserUpdater(
     mock(NewUserNotifier.class),
     db.getDbClient(),
-    mock(UserIndexer.class),
-    System2.INSTANCE,
+    userIndexer,
     organizationFlags,
     defaultOrganizationProvider,
     organizationCreation,
     new DefaultGroupFinder(db.getDbClient()),
     settings.asConfig());
+
   private UserIdentityAuthenticator underTest = new UserIdentityAuthenticator(db.getDbClient(), userUpdater, defaultOrganizationProvider, organizationFlags,
     new DefaultGroupFinder(db.getDbClient()));
 
   @Test
-  public void authenticate_new_user() throws Exception {
+  public void authenticate_new_user() {
     organizationFlags.setEnabled(true);
     underTest.authenticate(USER_IDENTITY, IDENTITY_PROVIDER, Source.realm(Method.BASIC, IDENTITY_PROVIDER.getName()));
 
@@ -111,7 +114,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void authenticate_new_user_with_groups() throws Exception {
+  public void authenticate_new_user_with_groups() {
     organizationFlags.setEnabled(true);
     GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1");
     GroupDto group2 = db.users().insertGroup(db.getDefaultOrganization(), "group2");
@@ -123,7 +126,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void authenticate_new_user_and_force_default_group_when_organizations_are_disabled() throws Exception {
+  public void authenticate_new_user_and_force_default_group_when_organizations_are_disabled() {
     organizationFlags.setEnabled(false);
     UserDto user = db.users().insertUser();
     GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1");
@@ -137,7 +140,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void does_not_force_default_group_when_authenticating_new_user_if_organizations_are_enabled() throws Exception {
+  public void does_not_force_default_group_when_authenticating_new_user_if_organizations_are_enabled() {
     organizationFlags.setEnabled(true);
     UserDto user = db.users().insertUser();
     GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1");
@@ -171,7 +174,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void authenticate_existing_user() throws Exception {
+  public void authenticate_existing_user() {
     db.users().insertUser(newUserDto()
       .setLogin(USER_LOGIN)
       .setActive(true)
@@ -192,7 +195,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void authenticate_existing_disabled_user() throws Exception {
+  public void authenticate_existing_disabled_user() {
     organizationFlags.setEnabled(true);
     db.users().insertUser(newUserDto()
       .setLogin(USER_LOGIN)
@@ -214,7 +217,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void authenticate_existing_user_and_add_new_groups() throws Exception {
+  public void authenticate_existing_user_and_add_new_groups() {
     organizationFlags.setEnabled(true);
     UserDto user = db.users().insertUser(newUserDto()
       .setLogin(USER_LOGIN)
@@ -229,7 +232,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void authenticate_existing_user_and_remove_groups() throws Exception {
+  public void authenticate_existing_user_and_remove_groups() {
     organizationFlags.setEnabled(true);
     UserDto user = db.users().insertUser(newUserDto()
       .setLogin(USER_LOGIN)
@@ -246,7 +249,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void authenticate_existing_user_and_remove_all_groups_expect_default_when_organizations_are_disabled() throws Exception {
+  public void authenticate_existing_user_and_remove_all_groups_expect_default_when_organizations_are_disabled() {
     organizationFlags.setEnabled(false);
     UserDto user = db.users().insertUser();
     GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1");
@@ -262,7 +265,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void does_not_force_default_group_when_authenticating_existing_user_when_organizations_are_enabled() throws Exception {
+  public void does_not_force_default_group_when_authenticating_existing_user_when_organizations_are_enabled() {
     organizationFlags.setEnabled(true);
     UserDto user = db.users().insertUser();
     GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1");
@@ -276,7 +279,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void ignore_groups_on_non_default_organizations() throws Exception {
+  public void ignore_groups_on_non_default_organizations() {
     organizationFlags.setEnabled(true);
     OrganizationDto org = db.organizations().insert();
     UserDto user = db.users().insertUser(newUserDto()
@@ -299,7 +302,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void fail_to_authenticate_new_user_when_allow_users_to_signup_is_false() throws Exception {
+  public void fail_to_authenticate_new_user_when_allow_users_to_signup_is_false() {
     TestIdentityProvider identityProvider = new TestIdentityProvider()
       .setKey("github")
       .setName("Github")
@@ -313,7 +316,7 @@ public class UserIdentityAuthenticatorTest {
   }
 
   @Test
-  public void fail_to_authenticate_new_user_when_email_already_exists() throws Exception {
+  public void fail_to_authenticate_new_user_when_email_already_exists() {
     db.users().insertUser(newUserDto()
       .setLogin("Existing user with same email")
       .setActive(true)

server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java

@@ -20,13 +20,21 @@
 package org.sonar.server.es;
 
 import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.stream.IntStream;
+import org.apache.commons.lang.math.RandomUtils;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
+import org.sonar.api.utils.internal.TestSystem2;
+import org.sonar.db.DbTester;
+import org.sonar.db.es.EsQueueDto;
 import org.sonar.server.es.BulkIndexer.Size;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -35,8 +43,12 @@ import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE;
 
 public class BulkIndexerTest {
 
+  private TestSystem2 testSystem2 = new TestSystem2().setNow(1_000L);
+
   @Rule
   public EsTester esTester = new EsTester(new FakeIndexDefinition().setReplicas(1));
+  @Rule
+  public DbTester dbTester = DbTester.create(testSystem2);
 
   @Test
   public void index_nothing() {
@@ -102,6 +114,42 @@ public class BulkIndexerTest {
     assertThat(count()).isEqualTo(removeFrom);
   }
 
+  @Test
+  @Ignore
+  public void when_index_is_done_EsQueues_must_be_deleted() {
+    BulkIndexer indexer = new BulkIndexer(esTester.client(), INDEX, Size.REGULAR);
+    int nbOfDelete = 10 + RandomUtils.nextInt(10);
+    int nbOfInsert = 10 + RandomUtils.nextInt(10);
+    int nbOfDocumentNotToBeDeleted = 10 + RandomUtils.nextInt(10);
+    Collection<EsQueueDto> esQueueDtos = new ArrayList<>();
+
+    // Those documents must be kept
+    FakeDoc[] docs = new FakeDoc[nbOfDocumentNotToBeDeleted];
+    for (int i = 1; i <= nbOfDocumentNotToBeDeleted; i++) {
+      docs[i] = FakeIndexDefinition.newDoc(-i);
+    }
+    esTester.putDocuments(INDEX_TYPE_FAKE, docs);
+
+    // Create nbOfDelete documents to be deleted
+    docs = new FakeDoc[nbOfDelete];
+    for (int i = 1; i <= nbOfDelete; i++) {
+      docs[i] = FakeIndexDefinition.newDoc(i);
+    }
+    esTester.putDocuments(INDEX_TYPE_FAKE, docs);
+    assertThat(count()).isEqualTo(nbOfDelete + nbOfDocumentNotToBeDeleted);
+
+    indexer.start(dbTester.getSession(), dbTester.getDbClient(), esQueueDtos);
+    // Create nbOfDelete for old Documents
+    IntStream.rangeClosed(1, nbOfDelete).forEach(
+      i -> indexer.addDeletion(INDEX_TYPE_FAKE, "" + i));
+    // Create nbOfInsert for new Documents
+    IntStream.rangeClosed(nbOfDelete + 1, nbOfInsert).forEach(
+      i -> indexer.add(newIndexRequest(i)));
+    indexer.stop();
+
+    assertThat(count()).isEqualTo(nbOfInsert + nbOfDocumentNotToBeDeleted);
+  }
+
   private long count() {
     return esTester.countDocuments("fakes", "fake");
   }

server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java

@@ -0,0 +1,394 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
+import org.junit.After;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonar.api.config.Settings;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.api.utils.internal.TestSystem2;
+import org.sonar.api.utils.log.LogTester;
+import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.db.DbSession;
+import org.sonar.db.DbTester;
+import org.sonar.db.es.EsQueueDto;
+import org.sonar.db.user.UserDto;
+import org.sonar.server.user.index.UserIndexDefinition;
+import org.sonar.server.user.index.UserIndexer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.sonar.api.utils.log.LoggerLevel.ERROR;
+import static org.sonar.api.utils.log.LoggerLevel.INFO;
+import static org.sonar.api.utils.log.LoggerLevel.TRACE;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+
+public class RecoveryIndexerTest {
+
+  private static final long PAST = 1_000L;
+  private TestSystem2 system2 = new TestSystem2().setNow(PAST);
+
+  @Rule
+  public final EsTester es = new EsTester(new UserIndexDefinition(new MapSettings().asConfig()));
+  @Rule
+  public final DbTester db = DbTester.create(system2);
+  @Rule
+  public final LogTester logTester = new LogTester().setLevel(TRACE);
+  @Rule
+  public TestRule safeguard = new Timeout(60, TimeUnit.SECONDS);
+
+  private RecoveryIndexer underTest;
+
+  @After
+  public void tearDown() {
+    if (underTest != null) {
+      underTest.stop();
+    }
+  }
+
+  @Test
+  public void display_default_configuration_at_startup() {
+    UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client());
+    underTest = newRecoveryIndexer(userIndexer, new MapSettings());
+
+    underTest.start();
+
+    assertThat(logTester.logs(LoggerLevel.DEBUG)).contains(
+      "Elasticsearch recovery - sonar.search.recovery.delayInMs=300000",
+      "Elasticsearch recovery - sonar.search.recovery.minAgeInMs=300000");
+  }
+
+  @Test
+  public void start_triggers_recovery_run_at_fixed_rate() throws Exception {
+    Settings settings = new MapSettings()
+      .setProperty("sonar.search.recovery.initialDelayInMs", "0")
+      .setProperty("sonar.search.recovery.delayInMs", "1");
+    underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mock(UserIndexer.class)));
+    AtomicInteger calls = new AtomicInteger(0);
+    doAnswer(invocation -> {
+      calls.incrementAndGet();
+      return null;
+    }).when(underTest).recover();
+
+    underTest.start();
+
+    // wait for 2 runs
+    while (calls.get() < 2) {
+      Thread.sleep(1L);
+    }
+  }
+
+  @Test
+  public void successfully_index_old_records() {
+    EsQueueDto item1 = createUnindexedUser();
+    EsQueueDto item2 = createUnindexedUser();
+
+    ProxyUserIndexer userIndexer = new ProxyUserIndexer();
+    advanceInTime();
+    underTest = newRecoveryIndexer(userIndexer);
+    underTest.recover();
+
+    assertThatQueueHasSize(0);
+    assertThat(userIndexer.called)
+      .extracting(EsQueueDto::getUuid)
+      .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid());
+
+    assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 2 USER");
+    assertThatLogsContain(INFO, "Elasticsearch recovery - 2 documents processed [0 failures]");
+  }
+
+  @Test
+  public void recent_records_are_not_recovered() {
+    createUnindexedUser();
+    createUnindexedUser();
+
+    ProxyUserIndexer userIndexer = new ProxyUserIndexer();
+    // do not advance in time
+    underTest = newRecoveryIndexer(userIndexer);
+    underTest.recover();
+
+    assertThatQueueHasSize(2);
+    assertThat(userIndexer.called).isEmpty();
+
+    assertThatLogsDoNotContain(TRACE, "Elasticsearch recovery - processing 2 USER");
+    assertThatLogsDoNotContain(INFO, "documents processed");
+  }
+
+  @Test
+  public void do_nothing_if_queue_is_empty() {
+    underTest = newRecoveryIndexer();
+
+    underTest.recover();
+
+    assertThatNoLogsFromRecovery(INFO);
+    assertThatNoLogsFromRecovery(ERROR);
+    assertThatQueueHasSize(0);
+  }
+
+  @Test
+  public void log_exception_on_recovery_failure() {
+    createUnindexedUser();
+    FailingOnceUserIndexer failingOnceUserIndexer = new FailingOnceUserIndexer();
+    advanceInTime();
+
+    underTest = newRecoveryIndexer(failingOnceUserIndexer);
+    underTest.recover();
+
+    // No rows treated
+    assertThatQueueHasSize(1);
+    assertThatLogsContain(ERROR, "Elasticsearch recovery - fail to recover documents");
+  }
+
+  @Test
+  public void scheduler_is_not_stopped_on_failures() throws Exception {
+    createUnindexedUser();
+    advanceInTime();
+    FailingUserIndexer userIndexer = new FailingUserIndexer();
+
+    underTest = newRecoveryIndexer(userIndexer);
+    underTest.start();
+
+    // all runs fail, but they are still scheduled
+    // -> waiting for 2 runs
+    while (userIndexer.called.size() < 2) {
+      Thread.sleep(1L);
+    }
+  }
+
+  @Test
+  public void recovery_retries_on_next_run_if_failure() throws Exception {
+    createUnindexedUser();
+    advanceInTime();
+    FailingOnceUserIndexer userIndexer = new FailingOnceUserIndexer();
+
+    underTest = newRecoveryIndexer(userIndexer);
+    underTest.start();
+
+    // first run fails, second run succeeds
+    userIndexer.counter.await(30, TimeUnit.SECONDS);
+
+    // First we expecting an exception at first run
+    // Then the second run must have treated all records
+    assertThatLogsContain(ERROR, "Elasticsearch recovery - fail to recover documents");
+    assertThatQueueHasSize(0);
+  }
+
+  @Test
+  public void stop_run_if_too_many_failures() throws Exception {
+    IntStream.range(0, 10).forEach(i -> createUnindexedUser());
+    advanceInTime();
+
+    // 10 docs to process, by groups of 3.
+    // The first group successfully recovers only 1 docs --> above 30% of failures --> stop run
+    PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(1);
+    Settings settings = new MapSettings()
+      .setProperty("sonar.search.recovery.loopLimit", "3");
+    underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings);
+    underTest.recover();
+
+    assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [2/3 documents], waiting for next run");
+    assertThatQueueHasSize(9);
+
+    // The indexer must have been called once and only once.
+    assertThat(failingAboveRatioUserIndexer.called).hasSize(3);
+  }
+
+  @Test
+  public void do_not_stop_run_if_success_rate_is_greater_than_ratio() throws Exception {
+    IntStream.range(0, 10).forEach(i -> createUnindexedUser());
+    advanceInTime();
+
+    // 10 docs to process, by groups of 5.
+    // Each group successfully recovers 4 docs --> below 30% of failures --> continue run
+    PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(4, 4, 2);
+    Settings settings = new MapSettings()
+      .setProperty("sonar.search.recovery.loopLimit", "5");
+    underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings);
+    underTest.recover();
+
+    assertThatLogsDoNotContain(ERROR, "too many failures");
+    assertThatQueueHasSize(0);
+    assertThat(failingAboveRatioUserIndexer.indexed).hasSize(10);
+    assertThat(failingAboveRatioUserIndexer.called).hasSize(10 + 2 /* retries */);
+  }
+
+  @Test
+  public void failing_always_on_same_document_does_not_generate_infinite_loop() {
+    EsQueueDto buggy = createUnindexedUser();
+    IntStream.range(0, 10).forEach(i -> createUnindexedUser());
+    advanceInTime();
+
+    FailingAlwaysOnSameElementIndexer indexer = new FailingAlwaysOnSameElementIndexer(buggy);
+    underTest = newRecoveryIndexer(indexer);
+    underTest.recover();
+
+    assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [1/1 documents], waiting for next run");
+    assertThatQueueHasSize(1);
+  }
+
+  private class ProxyUserIndexer extends UserIndexer {
+    private final List<EsQueueDto> called = new ArrayList<>();
+
+    ProxyUserIndexer() {
+      super(db.getDbClient(), es.client());
+    }
+
+    @Override
+    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+      called.addAll(items);
+      return super.index(dbSession, items);
+    }
+  }
+
+  private class FailingUserIndexer extends UserIndexer {
+    private final List<EsQueueDto> called = new ArrayList<>();
+
+    FailingUserIndexer() {
+      super(db.getDbClient(), es.client());
+    }
+
+    @Override
+    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+      called.addAll(items);
+      throw new RuntimeException("boom");
+    }
+
+  }
+
+  private class FailingOnceUserIndexer extends UserIndexer {
+    private final CountDownLatch counter = new CountDownLatch(2);
+
+    FailingOnceUserIndexer() {
+      super(db.getDbClient(), es.client());
+    }
+
+    @Override
+    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+      try {
+        if (counter.getCount() == 2) {
+          throw new RuntimeException("boom");
+        }
+        return super.index(dbSession, items);
+      } finally {
+        counter.countDown();
+      }
+    }
+  }
+
+  private class FailingAlwaysOnSameElementIndexer extends UserIndexer {
+    private final EsQueueDto failing;
+
+    FailingAlwaysOnSameElementIndexer(EsQueueDto failing) {
+      super(db.getDbClient(), es.client());
+      this.failing = failing;
+    }
+
+    @Override
+    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+      List<EsQueueDto> filteredItems = items.stream().filter(
+        i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList());
+      return super.index(dbSession, filteredItems);
+    }
+  }
+
+  private class PartiallyFailingUserIndexer extends UserIndexer {
+    private final List<EsQueueDto> called = new ArrayList<>();
+    private final List<EsQueueDto> indexed = new ArrayList<>();
+    private final Iterator<Integer> successfulReturns;
+
+    PartiallyFailingUserIndexer(int... successfulReturns) {
+      super(db.getDbClient(), es.client());
+      this.successfulReturns = IntStream.of(successfulReturns).iterator();
+    }
+
+    @Override
+    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+      System.out.println("called with " + items.size());
+      called.addAll(items);
+      int success = successfulReturns.next();
+      items.stream().limit(success).forEach(i -> {
+        System.out.println(" + success");
+        db.getDbClient().esQueueDao().delete(dbSession, i);
+        indexed.add(i);
+      });
+      dbSession.commit();
+      return success;
+    }
+  }
+
+  private void advanceInTime() {
+    system2.setNow(system2.now() + 100_000_000L);
+  }
+
+  private void assertThatLogsContain(LoggerLevel loggerLevel, String message) {
+    assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains(message)).isNotEmpty();
+  }
+
+  private void assertThatLogsDoNotContain(LoggerLevel loggerLevel, String message) {
+    assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains(message)).isEmpty();
+  }
+
+  private void assertThatNoLogsFromRecovery(LoggerLevel loggerLevel) {
+    assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains("Elasticsearch recovery - ")).isEmpty();
+  }
+
+  private void assertThatQueueHasSize(int number) {
+    assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isEqualTo(number);
+  }
+
+  private RecoveryIndexer newRecoveryIndexer() {
+    UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client());
+    return newRecoveryIndexer(userIndexer);
+  }
+
+  private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer) {
+    Settings settings = new MapSettings()
+      .setProperty("sonar.search.recovery.initialDelayInMs", "0")
+      .setProperty("sonar.search.recovery.delayInMs", "1")
+      .setProperty("sonar.search.recovery.minAgeInMs", "1");
+    return newRecoveryIndexer(userIndexer, settings);
+  }
+
+  private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, Settings settings) {
+    return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer);
+  }
+
+  private EsQueueDto createUnindexedUser() {
+    UserDto user = db.users().insertUser();
+    EsQueueDto item = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin());
+    db.getDbClient().esQueueDao().insert(db.getSession(), item);
+    db.commit();
+
+    return item;
+  }
+}

server/sonar-server/src/test/java/org/sonar/server/organization/OrganizationCreationImplTest.java

@@ -20,6 +20,7 @@
 package org.sonar.server.organization;
 
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import org.apache.commons.lang.RandomStringUtils;
@@ -113,7 +114,7 @@ public class OrganizationCreationImplTest {
   @Before
   public void setUp() {
     someUser = db.users().insertUser();
-    userIndexer.index(someUser.getLogin());
+    userIndexer.indexOnStartup(new HashSet<>());
   }
 
   @Test
@@ -263,10 +264,8 @@ public class OrganizationCreationImplTest {
   @Test
   public void create_add_current_user_as_member_of_organization() throws OrganizationCreation.KeyConflictException {
     UserDto user = db.users().insertUser();
-    userIndexer.index(user.getLogin());
-
     builtInQProfileRepositoryRule.initialize();
-    userIndexer.index(someUser.getLogin());
+    userIndexer.commitAndIndex(db.getSession(), someUser);
 
     OrganizationDto result = underTest.create(dbSession, someUser, FULL_POPULATED_NEW_ORGANIZATION);
 

server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java

@@ -21,6 +21,7 @@ package org.sonar.server.organization.ws;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -112,7 +113,7 @@ public class CreateActionTest {
   @Before
   public void setUp() {
     user = dbTester.users().insertUser();
-    userIndexer.index(user.getLogin());
+    userIndexer.indexOnStartup(new HashSet<>());
     userSession.logIn(user);
   }
 

server/sonar-server/src/test/java/org/sonar/server/organization/ws/DeleteActionTest.java

@@ -20,7 +20,6 @@
 
 package org.sonar.server.organization.ws;
 
-import java.util.Arrays;
 import java.util.List;
 import org.junit.Rule;
 import org.junit.Test;
@@ -59,6 +58,7 @@ import org.sonar.server.user.index.UserQuery;
 import org.sonar.server.ws.WsActionTester;
 
 import static com.google.common.collect.ImmutableList.of;
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
@@ -305,7 +305,7 @@ public class DeleteActionTest {
     db.organizations().addMember(org, user1);
     db.organizations().addMember(otherOrg, user1);
     db.organizations().addMember(org, user2);
-    userIndexer.index(Arrays.asList(user1.getLogin(), user2.getLogin()));
+    userIndexer.commitAndIndex(db.getSession(), asList(user1, user2));
     logInAsAdministrator(org);
 
     sendRequest(org);

server/sonar-server/src/test/java/org/sonar/server/organization/ws/RemoveMemberActionTest.java

@@ -20,6 +20,7 @@
 
 package org.sonar.server.organization.ws;
 
+import java.util.HashSet;
 import javax.annotation.Nullable;
 import org.junit.Before;
 import org.junit.Rule;
@@ -54,6 +55,7 @@ import org.sonar.server.ws.TestResponse;
 import org.sonar.server.ws.WsActionTester;
 
 import static java.net.HttpURLConnection.HTTP_NO_CONTENT;
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.groups.Tuple.tuple;
 import static org.sonar.api.CoreProperties.DEFAULT_ISSUE_ASSIGNEE;
@@ -93,11 +95,11 @@ public class RemoveMemberActionTest {
 
     user = db.users().insertUser();
     db.organizations().addMember(organization, user);
-    userIndexer.index(user.getLogin());
 
     UserDto adminUser = db.users().insertAdminByUserPermission(organization);
     db.organizations().addMember(organization, adminUser);
-    userIndexer.index(adminUser.getLogin());
+
+    userIndexer.indexOnStartup(new HashSet<>());
   }
 
   @Test
@@ -317,10 +319,9 @@ public class RemoveMemberActionTest {
     OrganizationDto anotherOrganization = db.organizations().insert();
     UserDto admin1 = db.users().insertAdminByUserPermission(anotherOrganization);
     db.organizations().addMember(anotherOrganization, admin1);
-    userIndexer.index(admin1.getLogin());
     UserDto admin2 = db.users().insertAdminByUserPermission(anotherOrganization);
     db.organizations().addMember(anotherOrganization, admin2);
-    userIndexer.index(admin2.getLogin());
+    userIndexer.commitAndIndex(db.getSession(), asList(admin1, admin2));
 
     call(anotherOrganization.getKey(), admin1.getLogin());
 

server/sonar-server/src/test/java/org/sonar/server/organization/ws/SearchMembersActionTest.java

@@ -127,7 +127,7 @@ public class SearchMembersActionTest {
   public void return_avatar() {
     UserDto user = db.users().insertUser(u -> u.setEmail("email@domain.com"));
     db.organizations().addMember(db.getDefaultOrganization(), user);
-    indexer.index(user.getLogin());
+    indexer.commitAndIndex(db.getSession(), user);
 
     SearchMembersWsResponse result = call();
 
@@ -195,7 +195,6 @@ public class SearchMembersActionTest {
     IntStream.range(0, 10).forEach(i -> {
       UserDto userDto = db.users().insertUser(user -> user.setName("USER_" + i));
       db.organizations().addMember(db.getDefaultOrganization(), userDto);
-      indexer.index(userDto.getLogin());
     });
     indexAllUsers();
     request.setPage(2).setPageSize(3);
@@ -214,7 +213,6 @@ public class SearchMembersActionTest {
     IntStream.range(0, 10).forEach(i -> {
       UserDto userDto = db.users().insertUser(user -> user.setName("USER_" + i));
       db.organizations().addMember(db.getDefaultOrganization(), userDto);
-      indexer.index(userDto.getLogin());
     });
     indexAllUsers();
     request.setQuery("_9");
@@ -229,7 +227,6 @@ public class SearchMembersActionTest {
     IntStream.range(0, 10).forEach(i -> {
       UserDto userDto = db.users().insertUser(user -> user.setLogin("USER_" + i));
       db.organizations().addMember(db.getDefaultOrganization(), userDto);
-      indexer.index(userDto.getLogin());
     });
     indexAllUsers();
     request.setQuery("_9");
@@ -246,7 +243,6 @@ public class SearchMembersActionTest {
         .setLogin("L" + i)
         .setEmail("USER_" + i + "@email.com"));
       db.organizations().addMember(db.getDefaultOrganization(), userDto);
-      indexer.index(userDto.getLogin());
     });
     indexAllUsers();
     request.setQuery("_9");

server/sonar-server/src/test/java/org/sonar/server/user/ServerUserSessionTest.java

@@ -484,7 +484,7 @@ public class ServerUserSessionTest {
 
     session.checkIsSystemAdministrator();
 
-    db.getDbClient().userDao().deactivateUserById(db.getSession(), user.getId());
+    db.getDbClient().userDao().deactivateUser(db.getSession(), user);
     db.commit();
 
     // should fail but succeeds because flag is kept in cache

server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java

@@ -67,8 +67,6 @@ public class UserIndexTest {
     assertThat(userDoc.email()).isEqualTo(user1.email());
     assertThat(userDoc.active()).isTrue();
     assertThat(userDoc.scmAccounts()).isEqualTo(user1.scmAccounts());
-    assertThat(userDoc.createdAt()).isEqualTo(user1.createdAt());
-    assertThat(userDoc.updatedAt()).isEqualTo(user1.updatedAt());
 
     assertThat(underTest.getNullableByLogin("")).isNull();
     assertThat(underTest.getNullableByLogin("unknown")).isNull();
@@ -202,9 +200,7 @@ public class UserIndexTest {
       .setName(login.toUpperCase(Locale.ENGLISH))
       .setEmail(login + "@mail.com")
       .setActive(true)
-      .setScmAccounts(scmAccounts)
-      .setCreatedAt(DATE_1)
-      .setUpdatedAt(DATE_2);
+      .setScmAccounts(scmAccounts);
   }
 
   private static UserDoc newUser(String login, String email, List<String> scmAccounts) {
@@ -213,8 +209,6 @@ public class UserIndexTest {
       .setName(login.toUpperCase(Locale.ENGLISH))
       .setEmail(email)
       .setActive(true)
-      .setScmAccounts(scmAccounts)
-      .setCreatedAt(DATE_1)
-      .setUpdatedAt(DATE_2);
+      .setScmAccounts(scmAccounts);
   }
 }

server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.server.user.index;
 
+import java.util.HashSet;
 import java.util.List;
 import org.junit.Rule;
 import org.junit.Test;
@@ -26,6 +27,7 @@ import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbTester;
 import org.sonar.db.user.UserDto;
+import org.sonar.db.user.UserTesting;
 import org.sonar.server.es.EsTester;
 
 import static java.util.Arrays.asList;
@@ -45,53 +47,34 @@ public class UserIndexerTest {
 
   @Test
   public void index_nothing_on_startup() {
-    underTest.indexOnStartup(null);
+    underTest.indexOnStartup(new HashSet<>());
 
     assertThat(es.countDocuments(UserIndexDefinition.INDEX_TYPE_USER)).isEqualTo(0L);
   }
 
   @Test
-  public void index_everything_on_startup() {
-    db.users().insertUser(user -> user
-      .setLogin("user1")
-      .setName("User1")
-      .setEmail("user1@mail.com")
-      .setActive(true)
-      .setScmAccounts(asList("user_1", "u1"))
-      .setCreatedAt(1500000000000L)
-      .setUpdatedAt(1500000000000L));
-
-    underTest.indexOnStartup(null);
+  public void indexOnStartup_adds_all_users_to_index() {
+    UserDto user = db.users().insertUser(u -> u
+      .setScmAccounts(asList("user_1", "u1")));
 
-    List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
-    assertThat(docs).hasSize(1);
-    UserDoc doc = docs.get(0);
-    assertThat(doc.login()).isEqualTo("user1");
-    assertThat(doc.name()).isEqualTo("User1");
-    assertThat(doc.email()).isEqualTo("user1@mail.com");
-    assertThat(doc.active()).isTrue();
-    assertThat(doc.scmAccounts()).containsOnly("user_1", "u1");
-    assertThat(doc.createdAt()).isEqualTo(1500000000000L);
-    assertThat(doc.updatedAt()).isEqualTo(1500000000000L);
-  }
-
-  @Test
-  public void index_single_user_on_startup() {
-    UserDto user = db.users().insertUser();
-
-    underTest.indexOnStartup(null);
+    underTest.indexOnStartup(new HashSet<>());
 
     List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
     assertThat(docs).hasSize(1);
-    assertThat(docs).extracting(UserDoc::login).containsExactly(user.getLogin());
+    UserDoc doc = docs.get(0);
+    assertThat(doc.login()).isEqualTo(user.getLogin());
+    assertThat(doc.name()).isEqualTo(user.getName());
+    assertThat(doc.email()).isEqualTo(user.getEmail());
+    assertThat(doc.active()).isEqualTo(user.isActive());
+    assertThat(doc.scmAccounts()).isEqualTo(user.getScmAccountsAsList());
   }
 
   @Test
-  public void index_single_user() {
+  public void commitAndIndex_single_user() {
     UserDto user = db.users().insertUser();
     UserDto anotherUser = db.users().insertUser();
 
-    underTest.index(user.getLogin());
+    underTest.commitAndIndex(db.getSession(), user);
 
     List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
     assertThat(docs).hasSize(1);
@@ -101,14 +84,14 @@ public class UserIndexerTest {
   }
 
   @Test
-  public void index_several_users() {
-    UserDto user = db.users().insertUser();
-    UserDto anotherUser = db.users().insertUser();
+  public void commitAndIndex_multiple_users() {
+    UserDto user1 = db.getDbClient().userDao().insert(db.getSession(), UserTesting.newUserDto());
+    UserDto user2 = db.getDbClient().userDao().insert(db.getSession(), UserTesting.newUserDto());
 
-    underTest.index(asList(user.getLogin(), anotherUser.getLogin()));
+    underTest.commitAndIndex(db.getSession(), asList(user1, user2));
 
     List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
-    assertThat(docs).hasSize(2);
-    assertThat(docs).extracting(UserDoc::login).containsOnly(user.getLogin(), anotherUser.getLogin());
+    assertThat(docs).extracting(UserDoc::login).containsExactlyInAnyOrder(user1.getLogin(), user2.getLogin());
+    assertThat(db.countRowsOfTable(db.getSession(), "users")).isEqualTo(2);
   }
 }

server/sonar-server/src/test/java/org/sonar/server/user/index/UserResultSetIteratorTest.java

@@ -1,101 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.user.index;
-
-import com.google.common.collect.Maps;
-import java.util.Arrays;
-import java.util.Map;
-import org.junit.Rule;
-import org.junit.Test;
-import org.sonar.db.DbTester;
-import org.sonar.db.organization.OrganizationDto;
-import org.sonar.db.user.UserDto;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class UserResultSetIteratorTest {
-
-  @Rule
-  public DbTester db = DbTester.create();
-
-  @Test
-  public void iterator_over_users() {
-    UserDto userDto1 = db.users().insertUser(u -> u
-      .setName("User1")
-      .setLogin("user1")
-      .setEmail("user1@mail.com")
-      .setScmAccounts(Arrays.asList("user_1", "u1"))
-      .setCreatedAt(1_500_000_000_000L)
-      .setUpdatedAt(1_500_000_000_000L));
-    UserDto userDto2 = db.users().insertUser(u -> u
-      .setName("User2")
-      .setLogin("user2")
-      .setEmail("user2@mail.com")
-      .setScmAccounts(Arrays.asList("user,2", "user_2"))
-      .setCreatedAt(1_500_000_000_000L)
-      .setUpdatedAt(1_500_000_000_000L));
-    UserDto inactiveUser = db.users().insertUser(u -> u
-      .setName("User3")
-      .setLogin("user3")
-      .setEmail(null)
-      .setActive(false)
-      .setScmAccounts((String) null)
-      .setCreatedAt(1_500_000_000_000L)
-      .setUpdatedAt(1_550_000_000_000L));
-    OrganizationDto org1 = db.organizations().insertForUuid("ORG_1");
-    OrganizationDto org2 = db.organizations().insertForUuid("ORG_2");
-    db.organizations().addMember(org1, userDto1);
-    db.organizations().addMember(org1, userDto2);
-    db.organizations().addMember(org2, userDto1);
-
-    UserResultSetIterator it = UserResultSetIterator.create(db.getDbClient(), db.getSession(), null);
-    Map<String, UserDoc> usersByLogin = Maps.uniqueIndex(it, UserDoc::login);
-    it.close();
-
-    assertThat(usersByLogin).hasSize(3);
-
-    UserDoc user1 = usersByLogin.get("user1");
-    assertThat(user1.name()).isEqualTo("User1");
-    assertThat(user1.email()).isEqualTo("user1@mail.com");
-    assertThat(user1.active()).isTrue();
-    assertThat(user1.scmAccounts()).containsOnly("user_1", "u1");
-    assertThat(user1.createdAt()).isEqualTo(1_500_000_000_000L);
-    assertThat(user1.updatedAt()).isEqualTo(1_500_000_000_000L);
-    assertThat(user1.organizationUuids()).containsOnly("ORG_1", "ORG_2");
-
-    UserDoc user2 = usersByLogin.get("user2");
-    assertThat(user2.name()).isEqualTo("User2");
-    assertThat(user2.email()).isEqualTo("user2@mail.com");
-    assertThat(user2.active()).isTrue();
-    assertThat(user2.scmAccounts()).containsOnly("user,2", "user_2");
-    assertThat(user2.createdAt()).isEqualTo(1_500_000_000_000L);
-    assertThat(user2.updatedAt()).isEqualTo(1_500_000_000_000L);
-    assertThat(user2.organizationUuids()).containsOnly("ORG_1");
-
-    UserDoc user3 = usersByLogin.get("user3");
-    assertThat(user3.name()).isEqualTo("User3");
-    assertThat(user3.email()).isNull();
-    assertThat(user3.active()).isFalse();
-    assertThat(user3.scmAccounts()).isEmpty();
-    assertThat(user3.createdAt()).isEqualTo(1500000000000L);
-    assertThat(user3.updatedAt()).isEqualTo(1550000000000L);
-    assertThat(user3.organizationUuids()).isEmpty();
-  }
-}

server/sonar-server/src/test/java/org/sonar/server/user/ws/ChangePasswordActionTest.java

@@ -24,7 +24,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.sonar.api.config.internal.MapSettings;
-import org.sonar.api.utils.System2;
 import org.sonar.db.DbTester;
 import org.sonar.server.es.EsTester;
 import org.sonar.server.exceptions.BadRequestException;
@@ -60,7 +59,6 @@ public class ChangePasswordActionTest {
   private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone();
 
   private UserUpdater userUpdater = new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), new UserIndexer(db.getDbClient(), esTester.client()),
-    System2.INSTANCE,
     organizationFlags,
     TestDefaultOrganizationProvider.from(db),
     mock(OrganizationCreation.class),
@@ -159,13 +157,15 @@ public class ChangePasswordActionTest {
   public void fail_to_update_password_on_external_auth() throws Exception {
     userSessionRule.logIn().setSystemAdministrator();
 
-    userUpdater.create(db.getSession(), NewUser.builder()
+    NewUser newUser = NewUser.builder()
       .setEmail("john@email.com")
       .setLogin("john")
       .setName("John")
       .setScmAccounts(newArrayList("jn"))
       .setExternalIdentity(new ExternalIdentity("gihhub", "john"))
-      .build());
+      .build();
+    userUpdater.createAndCommit(db.getSession(), newUser, u -> {
+    });
 
     expectedException.expect(BadRequestException.class);
     tester.newPostRequest("api/users", "change_password")
@@ -175,12 +175,13 @@ public class ChangePasswordActionTest {
   }
 
   private void createUser() {
-    userUpdater.create(db.getSession(), NewUser.builder()
+    userUpdater.createAndCommit(db.getSession(), NewUser.builder()
       .setEmail("john@email.com")
       .setLogin("john")
       .setName("John")
       .setScmAccounts(newArrayList("jn"))
       .setPassword("Valar Dohaeris")
-      .build());
+      .build(), u -> {
+      });
   }
 }

server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.server.user.ws;
 
+import java.util.HashSet;
 import java.util.Optional;
 import org.junit.Before;
 import org.junit.Rule;
@@ -81,11 +82,10 @@ public class CreateActionTest {
   private GroupDto defaultGroupInDefaultOrg;
   private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
   private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone();
-
   private OrganizationCreation organizationCreation = mock(OrganizationCreation.class);
   private WsActionTester tester = new WsActionTester(new CreateAction(
     db.getDbClient(),
-    new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), userIndexer, system2, organizationFlags, defaultOrganizationProvider,
+    new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), userIndexer, organizationFlags, defaultOrganizationProvider,
       organizationCreation, new DefaultGroupFinder(db.getDbClient()), settings.asConfig()),
     userSessionRule));
 
@@ -234,7 +234,7 @@ public class CreateActionTest {
     logInAsSystemAdministrator();
 
     db.users().insertUser(newUserDto("john", "John", "john@email.com").setActive(false));
-    userIndexer.index("john");
+    userIndexer.indexOnStartup(new HashSet<>());
 
     call(CreateRequest.builder()
       .setLogin("john")

server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java

@@ -334,14 +334,10 @@ public class DeactivateActionTest {
   }
 
   private UserDto insertUser(UserDto user) {
-    user
-      .setCreatedAt(system2.now())
-      .setUpdatedAt(system2.now());
     dbClient.userDao().insert(dbSession, user);
     dbClient.userTokenDao().insert(dbSession, newUserToken().setLogin(user.getLogin()));
     dbClient.propertiesDao().saveProperty(dbSession, new PropertyDto().setUserId(user.getId()).setKey("foo").setValue("bar"));
-    dbSession.commit();
-    userIndexer.index(user.getLogin());
+    userIndexer.commitAndIndex(dbSession, user);
     return user;
   }
 

server/sonar-server/src/test/java/org/sonar/server/user/ws/SearchActionTest.java

@@ -19,7 +19,7 @@
  */
 package org.sonar.server.user.ws;
 
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import org.junit.Rule;
@@ -281,8 +281,7 @@ public class SearchActionTest {
   }
 
   private List<UserDto> injectUsers(int numberOfUsers) throws Exception {
-    List<UserDto> userDtos = Lists.newArrayList();
-    long createdAt = System.currentTimeMillis();
+    List<UserDto> userDtos = new ArrayList<>();
     GroupDto group1 = db.users().insertGroup(newGroupDto().setName("sonar-users"));
     GroupDto group2 = db.users().insertGroup(newGroupDto().setName("sonar-admins"));
     for (int index = 0; index < numberOfUsers; index++) {
@@ -293,15 +292,13 @@ public class SearchActionTest {
 
       UserDto userDto = dbClient.userDao().insert(dbSession, new UserDto()
         .setActive(true)
-        .setCreatedAt(createdAt)
         .setEmail(email)
         .setLogin(login)
         .setName(name)
         .setScmAccounts(scmAccounts)
         .setLocal(true)
         .setExternalIdentity(login)
-        .setExternalIdentityProvider("sonarqube")
-        .setUpdatedAt(createdAt));
+        .setExternalIdentityProvider("sonarqube"));
       userDtos.add(userDto);
 
       for (int tokenIndex = 0; tokenIndex < index; tokenIndex++) {

server/sonar-server/src/test/java/org/sonar/server/user/ws/SkipOnboardingTutorialActionTest.java

@@ -23,7 +23,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.sonar.api.server.ws.WebService;
-import org.sonar.api.utils.internal.TestSystem2;
 import org.sonar.db.DbTester;
 import org.sonar.db.user.UserDto;
 import org.sonar.server.exceptions.UnauthorizedException;
@@ -35,9 +34,6 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 public class SkipOnboardingTutorialActionTest {
 
-  private final static long PAST = 100_000_000_000L;
-  private final static long NOW = 500_000_000_000L;
-
   @Rule
   public UserSessionRule userSession = UserSessionRule.standalone();
 
@@ -47,36 +43,31 @@ public class SkipOnboardingTutorialActionTest {
   @Rule
   public ExpectedException expectedException = ExpectedException.none();
 
-  private TestSystem2 system2 = new TestSystem2().setNow(NOW);
-
-  private WsActionTester ws = new WsActionTester(new SkipOnboardingTutorialAction(userSession, db.getDbClient(), system2));
+  private WsActionTester ws = new WsActionTester(new SkipOnboardingTutorialAction(userSession, db.getDbClient()));
 
   @Test
   public void mark_user_as_onboarded() {
     UserDto user = db.users().insertUser(u -> u
-      .setOnboarded(false)
-      .setUpdatedAt(PAST));
+      .setOnboarded(false));
     userSession.logIn(user);
 
     call();
 
     UserDto userDto = selectUser(user.getLogin());
     assertThat(userDto.isOnboarded()).isEqualTo(true);
-    assertThat(userDto.getUpdatedAt()).isEqualTo(NOW);
   }
 
   @Test
   public void does_nothing_if_user_already_onboarded() {
     UserDto user = db.users().insertUser(u -> u
-      .setOnboarded(true)
-      .setUpdatedAt(PAST));
+      .setOnboarded(true));
     userSession.logIn(user);
 
     call();
 
     UserDto userDto = selectUser(user.getLogin());
     assertThat(userDto.isOnboarded()).isEqualTo(true);
-    assertThat(userDto.getUpdatedAt()).isEqualTo(PAST);
+    assertThat(userDto.getUpdatedAt()).isEqualTo(user.getUpdatedAt());
   }
 
   @Test
@@ -112,7 +103,6 @@ public class SkipOnboardingTutorialActionTest {
 
   @Test
   public void test_definition() {
-    WsActionTester ws = new WsActionTester(new SkipOnboardingTutorialAction(userSession, db.getDbClient(), system2));
     WebService.Action def = ws.getDef();
     assertThat(def.isPost()).isTrue();
     assertThat(def.isInternal()).isTrue();

server/sonar-server/src/test/java/org/sonar/server/user/ws/UpdateActionTest.java

@@ -75,7 +75,7 @@ public class UpdateActionTest {
     dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users");
     userIndexer = new UserIndexer(dbClient, esTester.client());
     tester = new WsTester(new UsersWs(new UpdateAction(
-      new UserUpdater(mock(NewUserNotifier.class), dbClient, userIndexer, system2, organizationFlags, defaultOrganizationProvider, ORGANIZATION_CREATION_NOT_USED_FOR_UPDATE,
+      new UserUpdater(mock(NewUserNotifier.class), dbClient, userIndexer, organizationFlags, defaultOrganizationProvider, ORGANIZATION_CREATION_NOT_USED_FOR_UPDATE,
         new DefaultGroupFinder(dbTester.getDbClient()), settings.asConfig()),
       userSessionRule,
       new UserJsonWriter(userSessionRule), dbClient)));
@@ -226,7 +226,6 @@ public class UpdateActionTest {
       .setExternalIdentity("jo")
       .setExternalIdentityProvider("sonarqube");
     dbClient.userDao().insert(session, userDto);
-    session.commit();
-    userIndexer.index(userDto.getLogin());
+    userIndexer.commitAndIndex(session, userDto);
   }
 }

tests/pom.xml

@@ -157,6 +157,34 @@
           </includes>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-byteman-for-resilience-tests</id>
+            <phase>generate-test-resources</phase>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>org.jboss.byteman</groupId>
+                  <artifactId>byteman</artifactId>
+                  <version>3.0.10</version>
+                  <overWrite>false</overWrite>
+                  <destFileName>byteman.jar</destFileName>
+                </artifactItem>
+              </artifactItems>
+              <outputDirectory>${project.basedir}/target</outputDirectory>
+              <overWriteReleases>false</overWriteReleases>
+              <overWriteSnapshots>false</overWriteSnapshots>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
     </plugins>
   </build>
 

tests/resilience/user_indexer.btm

@@ -0,0 +1,23 @@
+# sonar.web.javaAdditionalOpts=-javaagent:/path/to/byteman-3.0.10/lib/byteman.jar=script:/path/to/user_indexer.btm,boot:/path/to/byteman-3.0.10/lib/byteman.jar
+# sonar.search.recovery.delayInMs=10000
+# sonar.search.recovery.minAgeInMs=30000
+
+RULE make indexing of users silently fail
+CLASS org.sonar.server.user.index.UserIndexer
+METHOD postCommit
+COMPILE
+AT ENTRY
+BIND logins:Collection = $logins
+IF logins.contains("error")
+DO RETURN
+ENDRULE
+
+RULE make indexing of users fail
+CLASS org.sonar.server.user.index.UserIndexer
+METHOD postCommit
+COMPILE
+AT ENTRY
+BIND logins:Collection = $logins
+IF logins.contains("crash")
+DO THROW new RuntimeException("Fail to index users to Elasticsearch because a user 'crash' has been given")
+ENDRULE

tests/src/test/java/org/sonarqube/tests/Category5Suite.java

@@ -31,6 +31,7 @@ import org.sonarqube.tests.updateCenter.UpdateCenterTest;
 import org.sonarqube.tests.user.OnboardingTest;
 import org.sonarqube.tests.user.RealmAuthenticationTest;
 import org.sonarqube.tests.user.SsoAuthenticationTest;
+import org.sonarqube.tests.user.UserEsResilienceTest;
 
 /**
  * This suite is reserved to the tests that start their own instance of Orchestrator.
@@ -49,7 +50,8 @@ import org.sonarqube.tests.user.SsoAuthenticationTest;
   RealmAuthenticationTest.class,
   SsoAuthenticationTest.class,
   OnboardingTest.class,
-  BuiltInQualityProfilesNotificationTest.class
+  BuiltInQualityProfilesNotificationTest.class,
+  UserEsResilienceTest.class
 })
 public class Category5Suite {
 

tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java

@@ -0,0 +1,144 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.tests.user;
+
+import com.sonar.orchestrator.Orchestrator;
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.WsUsers.CreateWsResponse.User;
+import org.sonarqube.ws.client.user.SearchRequest;
+import org.sonarqube.ws.client.user.UpdateRequest;
+
+import static java.lang.String.format;
+import static org.assertj.core.api.Assertions.assertThat;
+import static util.ItUtils.expectHttpError;
+
+public class UserEsResilienceTest {
+
+  @ClassRule
+  public static final Orchestrator orchestrator = Orchestrator.builderEnv()
+    .setServerProperty("sonar.web.javaAdditionalOpts",
+      format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar()))
+    .setServerProperty("sonar.search.recovery.delayInMs", "1000")
+    .setServerProperty("sonar.search.recovery.minAgeInMs", "3000")
+    .build();
+
+  @Rule
+  public TestRule timeout = new DisableOnDebug(Timeout.builder()
+    .withLookingForStuckThread(true)
+    .withTimeout(60L, TimeUnit.SECONDS)
+    .build());
+
+  @Rule
+  public Tester tester = new Tester(orchestrator);
+
+  @Test
+  public void creation_and_update_of_user_are_resilient_to_indexing_errors() throws Exception {
+    String login = "error";
+
+    // creation of user succeeds but index is not up-to-date (indexing
+    // failures are not propagated to web services)
+    User user = tester.users().generate(u -> u.setLogin(login));
+
+    // user exists in db, it can't be created again.
+    // However he's not indexed.
+    expectHttpError(400, "An active user with login '" + login + "' already exists",
+      () -> tester.users().generate(u -> u.setLogin(login)));
+    assertThat(isReturnedInSearch(user.getLogin())).isFalse();
+
+    while (!isReturnedInSearch(user.getLogin())) {
+      // user is indexed by the recovery daemon, which runs every 5 seconds
+      Thread.sleep(1_000L);
+    }
+
+    // update the name. Db operation succeeds but not ES indexing.
+    // Renaming is not propagated to index as long as recovery does not
+    // run.
+    String newName = "renamed";
+    tester.users().service().update(UpdateRequest.builder().setLogin(login).setName(newName).build());
+    assertThat(isReturnedInSearch(newName)).isFalse();
+
+    while (!isReturnedInSearch(newName)) {
+      // user is indexed by the recovery daemon, which runs every 5 seconds
+      Thread.sleep(1_000L);
+    }
+  }
+
+  @Test
+  public void creation_and_update_of_user_are_resilient_to_indexing_crash() throws Exception {
+    String login = "crash";
+
+    // creation of user succeeds but index is not up-to-date (indexing
+    // crashes are not propagated to web services)
+    expectHttpError(500, () -> tester.users().generate(u -> u.setLogin(login)));
+
+    // user exists in db, it can't be created again.
+    // However he's not indexed.
+    expectHttpError(400, "An active user with login '" + login + "' already exists",
+      () -> tester.users().generate(u -> u.setLogin(login)));
+    assertThat(isReturnedInSearch(login)).isFalse();
+
+    while (!isReturnedInSearch(login)) {
+      // user is indexed by the recovery daemon, which runs every 5 seconds
+      Thread.sleep(1_000L);
+    }
+
+    // update the name. Db operation succeeds but ES indexing crashes.
+    // Renaming is not propagated to index as long as recovery does not
+    // run.
+    String newName = "renamed";
+    expectHttpError(500, () -> tester.users().service().update(UpdateRequest.builder().setLogin(login).setName(newName).build()));
+    assertThat(isReturnedInSearch(newName)).isFalse();
+
+    while (!isReturnedInSearch(newName)) {
+      // user is indexed by the recovery daemon, which runs every 5 seconds
+      Thread.sleep(1_000L);
+    }
+  }
+
+  private boolean isReturnedInSearch(String name) {
+    return tester.users().service().search(SearchRequest.builder().setQuery(name).build()).getUsersCount() == 1L;
+  }
+
+  private static String findBytemanJar() {
+    // see pom.xml, Maven copies and renames the artifact.
+    File jar = new File("target/byteman.jar");
+    if (!jar.exists()) {
+      throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once.");
+    }
+    return jar.getAbsolutePath();
+  }
+
+  private static String findBytemanScript() {
+    // see pom.xml, Maven copies and renames the artifact.
+    File script = new File("resilience/user_indexer.btm");
+    if (!script.exists()) {
+      throw new IllegalStateException("Can't find " + script);
+    }
+    return script.getAbsolutePath();
+  }
+}