From 476c36e8358eadf57fe1821a0cdd2ec708dab7a3 Mon Sep 17 00:00:00 2001 From: Teryk Bellahsene Date: Wed, 22 Mar 2017 14:55:24 +0100 Subject: [PATCH] SONAR-8980 Index a list of users --- .../sonar/server/user/index/UserIndexer.java | 42 ++++++++++++----- .../user/index/UserResultSetIterator.java | 42 +++++++++++------ .../server/user/index/UserIndexerTest.java | 45 ++++++++++++++----- 3 files changed, 94 insertions(+), 35 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 34c906db284..e8c535915b3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -21,6 +21,7 @@ package org.sonar.server.user.index; import com.google.common.collect.ImmutableSet; import java.util.Iterator; +import java.util.List; import java.util.Set; import javax.annotation.Nullable; import org.elasticsearch.action.index.IndexRequest; @@ -32,7 +33,9 @@ import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.StartupIndexer; +import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; +import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput; import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER; public class UserIndexer implements StartupIndexer { @@ -52,35 +55,52 @@ public class UserIndexer implements StartupIndexer { @Override public void indexOnStartup(Set emptyIndexTypes) { - doIndex(null, Size.LARGE); + doIndex(newBulkIndexer(Size.LARGE), null); } public void index(String login) { requireNonNull(login); - doIndex(login, Size.REGULAR); + doIndex(newBulkIndexer(Size.REGULAR), singletonList(login)); } - private void doIndex(@Nullable String login, Size bulkSize) { - final BulkIndexer bulk = new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex()); - bulk.setSize(bulkSize); + public void index(List logins) { + requireNonNull(logins); + if (logins.isEmpty()) { + return; + } + doIndex(newBulkIndexer(Size.REGULAR), logins); + } + + private void doIndex(BulkIndexer bulk, @Nullable List logins) { try (DbSession dbSession = dbClient.openSession(false)) { - try (UserResultSetIterator rowIt = UserResultSetIterator.create(dbClient, dbSession, login)) { - doIndex(bulk, rowIt); + if (logins == null) { + processLogins(bulk, dbSession, null); + } else { + executeLargeInputsWithoutOutput(logins, l -> processLogins(bulk, dbSession, l)); } } } - private static long doIndex(BulkIndexer bulk, Iterator users) { - long maxUpdatedAt = 0L; + private void processLogins(BulkIndexer bulk, DbSession dbSession, @Nullable List logins) { + try (UserResultSetIterator rowIt = UserResultSetIterator.create(dbClient, dbSession, logins)) { + processResultSet(bulk, rowIt); + } + } + + private static void processResultSet(BulkIndexer bulk, Iterator users) { bulk.start(); while (users.hasNext()) { UserDoc user = users.next(); bulk.add(newIndexRequest(user)); - maxUpdatedAt = Math.max(maxUpdatedAt, user.updatedAt()); } bulk.stop(); - return maxUpdatedAt; + } + + private BulkIndexer newBulkIndexer(Size bulkSize) { + final BulkIndexer bulk = new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex()); + bulk.setSize(bulkSize); + return bulk; } private static IndexRequest newIndexRequest(UserDoc user) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java index f2876942d74..dc1a707310d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java @@ -19,12 +19,15 @@ */ package org.sonar.server.user.index; +import com.google.common.base.Joiner; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; import org.sonar.db.DbClient; @@ -32,8 +35,6 @@ import org.sonar.db.DbSession; import org.sonar.db.ResultSetIterator; import org.sonar.db.user.UserDto; -import static java.util.Collections.singletonList; - /** * Scrolls over table USERS and reads documents to populate the user index */ @@ -51,7 +52,8 @@ class UserResultSetIterator extends ResultSetIterator { }; private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from users u "; - private static final String LOGIN_FILTER = " WHERE u.login=?"; + private static final String LOGIN_FILTER = "u.login=?"; + private static final Joiner OR_JOINER = Joiner.on(" or "); private final ListMultimap organizationUuidsByLogins; @@ -60,17 +62,17 @@ class UserResultSetIterator extends ResultSetIterator { this.organizationUuidsByLogins = organizationUuidsByLogins; } - static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable String login) { + static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable List logins) { try { - String sql = createSql(login); + String sql = createSql(logins); PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); - setParameter(stmt, login); + setParameters(stmt, logins); ListMultimap organizationUuidsByLogin = ArrayListMultimap.create(); - if (login == null) { + if (logins == null) { dbClient.organizationMemberDao().selectAllForUserIndexing(session, organizationUuidsByLogin::put); } else { - dbClient.organizationMemberDao().selectForUserIndexing(session, singletonList(login), organizationUuidsByLogin::put); + dbClient.organizationMemberDao().selectForUserIndexing(session, logins, organizationUuidsByLogin::put); } return new UserResultSetIterator(stmt, organizationUuidsByLogin); @@ -79,15 +81,29 @@ class UserResultSetIterator extends ResultSetIterator { } } - private static String createSql(@Nullable String login) { + private static String createSql(@Nullable List logins) { + if (logins == null) { + return SQL_ALL; + } + + List sqlLogins = logins.stream() + .map(l -> LOGIN_FILTER) + .collect(Collectors.toList()); + String sql = SQL_ALL; - sql += login == null ? "" : LOGIN_FILTER; + sql += " WHERE "; + sql += "(" + OR_JOINER.join(sqlLogins) + ")"; + return sql; } - private static void setParameter(PreparedStatement stmt, @Nullable String login) throws SQLException { - if (login != null) { - stmt.setString(1, login); + private static void setParameters(PreparedStatement stmt, @Nullable List logins) throws SQLException { + if (logins == null) { + return; + } + + for (int i = 0; i < logins.size(); i++) { + stmt.setString(i + 1, logins.get(i)); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java index 8f9230c55ee..4726c05b828 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java @@ -19,6 +19,7 @@ */ package org.sonar.server.user.index; +import java.util.Arrays; import java.util.List; import org.junit.Rule; import org.junit.Test; @@ -40,19 +41,20 @@ public class UserIndexerTest { @Rule public EsTester es = new EsTester(new UserIndexDefinition(new MapSettings())); + private UserIndexer underTest = new UserIndexer(db.getDbClient(), es.client()); + @Test - public void index_nothing() { - UserIndexer indexer = createIndexer(); - indexer.indexOnStartup(null); + public void index_nothing_on_startup() { + underTest.indexOnStartup(null); + assertThat(es.countDocuments(UserIndexDefinition.INDEX_TYPE_USER)).isEqualTo(0L); } @Test - public void index_everything() { + public void index_everything_on_startup() { db.prepareDbUnit(getClass(), "index.xml"); - UserIndexer indexer = createIndexer(); - indexer.indexOnStartup(null); + underTest.indexOnStartup(null); List docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); assertThat(docs).hasSize(1); @@ -67,18 +69,39 @@ public class UserIndexerTest { } @Test - public void index_single_user() { + public void index_single_user_on_startup() { UserDto user = db.users().insertUser(); - UserIndexer indexer = createIndexer(); - indexer.indexOnStartup(null); + underTest.indexOnStartup(null); List docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); assertThat(docs).hasSize(1); assertThat(docs).extracting(UserDoc::login).containsExactly(user.getLogin()); } - private UserIndexer createIndexer() { - return new UserIndexer(db.getDbClient(), es.client()); + @Test + public void index_single_user() { + UserDto user = db.users().insertUser(); + UserDto anotherUser = db.users().insertUser(); + + underTest.index(user.getLogin()); + + List docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); + assertThat(docs).hasSize(1); + assertThat(docs).extracting(UserDoc::login) + .containsExactly(user.getLogin()) + .doesNotContain(anotherUser.getLogin()); + } + + @Test + public void index_several_users() { + UserDto user = db.users().insertUser(); + UserDto anotherUser = db.users().insertUser(); + + underTest.index(Arrays.asList(user.getLogin(), anotherUser.getLogin())); + + List docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); + assertThat(docs).hasSize(2); + assertThat(docs).extracting(UserDoc::login).containsOnly(user.getLogin(), anotherUser.getLogin()); } } -- 2.39.5