diff options
author | Daniel Schwarz <daniel.schwarz@sonarsource.com> | 2017-02-24 14:55:02 +0100 |
---|---|---|
committer | Daniel Schwarz <bartfastiel@users.noreply.github.com> | 2017-02-24 17:21:23 +0100 |
commit | 075a9ff5026d0cf04ef16bb0c9c18de43614c1cd (patch) | |
tree | c8a83bcae1d6a7378eab0e46ca650734f679a2a8 | |
parent | 100bbe4a185e9da41bf444caa454eaa9ddd229af (diff) | |
download | sonarqube-075a9ff5026d0cf04ef16bb0c9c18de43614c1cd.tar.gz sonarqube-075a9ff5026d0cf04ef16bb0c9c18de43614c1cd.zip |
SOANR-8092 make BulkIndexer size an enum for better readability
11 files changed, 65 insertions, 47 deletions
diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index e5785c8367d..e5036520f7f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -37,6 +37,7 @@ import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.component.ComponentDto; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.ProjectIndexer; @@ -69,7 +70,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe @Override public void indexOnStartup(Set<IndexType> emptyIndexTypes) { - doIndexByProjectUuid(null, true); + doIndexByProjectUuid(null, Size.LARGE); } @Override @@ -78,7 +79,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe case PROJECT_CREATION: case PROJECT_KEY_UPDATE: case NEW_ANALYSIS: - doIndexByProjectUuid(projectUuid, false); + doIndexByProjectUuid(projectUuid, Size.REGULAR); break; default: // defensive case @@ -95,9 +96,9 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe * @param projectUuid the uuid of the project to analyze, or <code>null</code> if all content should be indexed.<br/> * <b>Warning:</b> only use <code>null</code> during startup. */ - private void doIndexByProjectUuid(@Nullable String projectUuid, boolean largeBulkIndexing) { + private void doIndexByProjectUuid(@Nullable String projectUuid, Size bulkSize) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT.getIndex()); - bulk.setLarge(largeBulkIndexing); + bulk.setSize(bulkSize); bulk.start(); try (DbSession dbSession = dbClient.openSession(false)) { @@ -136,7 +137,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe private void indexNow(ComponentDto... docs) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT.getIndex()); - bulk.setLarge(false); + bulk.setSize(Size.REGULAR); bulk.start(); Arrays.stream(docs) .map(ComponentIndexer::toDocument) diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index a946f5a6cf0..4679c63a70e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -66,7 +66,7 @@ public class BulkIndexer implements Startable { private final EsClient client; private final String indexName; - private boolean large = false; + private Size size = Size.REGULAR; private long flushByteSize = FLUSH_BYTE_SIZE; private BulkRequestBuilder bulkRequest = null; private Map<String, Object> largeInitialSettings = null; @@ -86,13 +86,21 @@ public class BulkIndexer implements Startable { this.semaphore = new Semaphore(concurrentRequests); } + public enum Size { + /** Use this size for a limited number of documents. */ + REGULAR, + + /** Use this size for initial indexing and if you expect unusual huge numbers of documents. */ + LARGE; + } + /** * Large indexing is an heavy operation that populates an index generally from scratch. Replicas and * automatic refresh are disabled during bulk indexing and lucene segments are optimized at the end. */ - public BulkIndexer setLarge(boolean b) { + public BulkIndexer setSize(Size size) { Preconditions.checkState(bulkRequest == null, ALREADY_STARTED_MESSAGE); - this.large = b; + this.size = size; return this; } @@ -104,7 +112,7 @@ public class BulkIndexer implements Startable { @Override public void start() { Preconditions.checkState(bulkRequest == null, ALREADY_STARTED_MESSAGE); - if (large) { + if (size == Size.LARGE) { largeInitialSettings = Maps.newHashMap(); Map<String, Object> bulkSettings = Maps.newHashMap(); GetSettingsResponse settingsResp = client.nativeClient().admin().indices().prepareGetSettings(indexName).get(); @@ -204,7 +212,7 @@ public class BulkIndexer implements Startable { } progress.stop(); client.prepareRefresh(indexName).get(); - if (large) { + if (size == Size.LARGE) { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 2d76023a937..cae8a0aa8cb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.sonar.api.resources.Qualifiers; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexType; @@ -69,7 +70,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S @Override public void indexOnStartup(Set<IndexType> emptyIndexTypes) { - doIndex(createBulkIndexer(true), (String) null); + doIndex(createBulkIndexer(Size.LARGE), (String) null); } @Override @@ -81,7 +82,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S // nothing to do, project key is not used in this index break; case NEW_ANALYSIS: - doIndex(createBulkIndexer(false), projectUuid); + doIndex(createBulkIndexer(Size.REGULAR), projectUuid); break; default: // defensive case @@ -93,11 +94,11 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S * For benchmarks */ public void index(Iterator<IssueDoc> issues) { - doIndex(createBulkIndexer(false), issues); + doIndex(createBulkIndexer(Size.REGULAR), issues); } public void index(Collection<String> issueKeys) { - doIndex(createBulkIndexer(false), issueKeys); + doIndex(createBulkIndexer(Size.REGULAR), issueKeys); } private void doIndex(BulkIndexer bulk, Collection<String> issueKeys) { @@ -154,9 +155,9 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S esClient.prepareRefresh(INDEX_TYPE_ISSUE.getIndex()).get(); } - private BulkIndexer createBulkIndexer(boolean largeBulkIndexing) { + private BulkIndexer createBulkIndexer(Size bulkSize) { return new BulkIndexer(esClient, INDEX_TYPE_ISSUE.getIndex()) - .setLarge(largeBulkIndexing); + .setSize(bulkSize); } private static IndexRequest newIndexRequest(IssueDoc issue) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index 82b47d2dda8..8ba20bb0dfb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -31,6 +31,7 @@ import org.sonar.db.DbSession; import org.sonar.db.measure.ProjectMeasuresIndexerIterator; import org.sonar.db.measure.ProjectMeasuresIndexerIterator.ProjectMeasures; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.ProjectIndexer; @@ -59,7 +60,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization @Override public void indexOnStartup(Set<IndexType> emptyIndexTypes) { - doIndex(createBulkIndexer(true), (String) null); + doIndex(createBulkIndexer(Size.LARGE), (String) null); } @Override @@ -75,7 +76,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization case PROJECT_CREATION: // provisioned projects are supported by WS api/components/search_projects case NEW_ANALYSIS: - doIndex(createBulkIndexer(false), projectUuid); + doIndex(createBulkIndexer(Size.REGULAR), projectUuid); break; default: // defensive case @@ -108,9 +109,9 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization bulk.stop(); } - private BulkIndexer createBulkIndexer(boolean largeBulkIndexing) { + private BulkIndexer createBulkIndexer(Size bulkSize) { return new BulkIndexer(esClient, INDEX_TYPE_PROJECT_MEASURES.getIndex()) - .setLarge(largeBulkIndexing); + .setSize(bulkSize); } private static IndexRequest newIndexRequest(ProjectMeasuresDoc doc) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java index 8c0c25e17cb..c938b83521c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java @@ -41,6 +41,7 @@ import org.sonar.core.util.stream.Collectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.ProjectIndexer; @@ -93,7 +94,7 @@ public class PermissionIndexer implements ProjectIndexer, Startable, StartupInde Future<?> submit = executor.submit(() -> { List<Dto> authorizations = getAllAuthorizations(); Stream<AuthorizationScope> scopes = getScopes(emptyIndexTypes); - index(authorizations, scopes, true); + index(authorizations, scopes, Size.LARGE); }); try { Uninterruptibles.getUninterruptibly(submit); @@ -117,7 +118,7 @@ public class PermissionIndexer implements ProjectIndexer, Startable, StartupInde @VisibleForTesting void index(List<Dto> authorizations) { - index(authorizations, authorizationScopes.stream(), false); + index(authorizations, authorizationScopes.stream(), Size.REGULAR); } @Override @@ -151,20 +152,20 @@ public class PermissionIndexer implements ProjectIndexer, Startable, StartupInde .filter(scope -> indexTypes.contains(scope.getIndexType())); } - private void index(Collection<PermissionIndexerDao.Dto> authorizations, Stream<AuthorizationScope> scopes, boolean largeBulkIndexing) { + private void index(Collection<PermissionIndexerDao.Dto> authorizations, Stream<AuthorizationScope> scopes, Size bulkSize) { if (authorizations.isEmpty()) { return; } // index each authorization in each scope - scopes.forEach(scope -> index(authorizations, scope, largeBulkIndexing)); + scopes.forEach(scope -> index(authorizations, scope, bulkSize)); } - private void index(Collection<PermissionIndexerDao.Dto> authorizations, AuthorizationScope scope, boolean largeBulkIndexing) { + private void index(Collection<PermissionIndexerDao.Dto> authorizations, AuthorizationScope scope, Size bulkSize) { IndexType indexType = scope.getIndexType(); BulkIndexer bulkIndexer = new BulkIndexer(esClient, indexType.getIndex()); - bulkIndexer.setLarge(largeBulkIndexing); + bulkIndexer.setSize(bulkSize); bulkIndexer.start(); authorizations.stream() diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java index da74a705a3b..0d7b437546c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java @@ -34,6 +34,7 @@ import org.sonar.db.DbSession; import org.sonar.db.qualityprofile.ActiveRuleKey; import org.sonar.server.es.BaseIndexer; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.qualityprofile.ActiveRuleChange; @@ -54,11 +55,11 @@ public class ActiveRuleIndexer extends BaseIndexer { @Override protected long doIndex(long lastUpdatedAt) { - return doIndex(createBulkIndexer(false), lastUpdatedAt); + return doIndex(createBulkIndexer(Size.REGULAR), lastUpdatedAt); } public void index(Iterator<ActiveRuleDoc> rules) { - doIndex(createBulkIndexer(false), rules); + doIndex(createBulkIndexer(Size.REGULAR), rules); } private long doIndex(BulkIndexer bulk, long lastUpdatedAt) { @@ -114,9 +115,9 @@ public class ActiveRuleIndexer extends BaseIndexer { bulk.stop(); } - private BulkIndexer createBulkIndexer(boolean large) { + private BulkIndexer createBulkIndexer(Size size) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex()); - bulk.setLarge(large); + bulk.setSize(size); return bulk; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java index def1b40e20d..1bff020d964 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java @@ -26,6 +26,7 @@ import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.server.es.BaseIndexer; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_UPDATED_AT; @@ -42,11 +43,11 @@ public class RuleIndexer extends BaseIndexer { @Override protected long doIndex(long lastUpdatedAt) { - return doIndex(createBulkIndexer(false), lastUpdatedAt); + return doIndex(createBulkIndexer(Size.REGULAR), lastUpdatedAt); } public void index(Iterator<RuleDoc> rules) { - doIndex(createBulkIndexer(false), rules); + doIndex(createBulkIndexer(Size.REGULAR), rules); } private long doIndex(BulkIndexer bulk, long lastUpdatedAt) { @@ -76,9 +77,9 @@ public class RuleIndexer extends BaseIndexer { return maxDate; } - private BulkIndexer createBulkIndexer(boolean large) { + private BulkIndexer createBulkIndexer(Size size) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_RULE.getIndex()); - bulk.setLarge(large); + bulk.setSize(size); return bulk; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java index 9ab3c90f718..c647afae7c3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.ProjectIndexer; @@ -61,7 +62,7 @@ public class TestIndexer implements ProjectIndexer, StartupIndexer { break; case NEW_ANALYSIS: deleteProject(projectUuid); - doIndex(projectUuid, false); + doIndex(projectUuid, Size.REGULAR); break; default: // defensive case @@ -76,7 +77,7 @@ public class TestIndexer implements ProjectIndexer, StartupIndexer { @Override public void indexOnStartup(Set<IndexType> emptyIndexTypes) { - doIndex(null, true); + doIndex(null, Size.LARGE); } public long index(Iterator<FileSourcesUpdaterHelper.Row> dbRows) { @@ -84,9 +85,9 @@ public class TestIndexer implements ProjectIndexer, StartupIndexer { return doIndex(bulk, dbRows); } - private long doIndex(@Nullable String projectUuid, boolean largeBulkIndexing) { + private long doIndex(@Nullable String projectUuid, Size bulkSize) { final BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_TEST.getIndex()); - bulk.setLarge(largeBulkIndexing); + bulk.setSize(bulkSize); DbSession dbSession = dbClient.openSession(false); try { diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 65a5c848519..edac71f8fdc 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.StartupIndexer; @@ -51,17 +52,17 @@ public class UserIndexer implements StartupIndexer { @Override public void indexOnStartup(Set<IndexType> emptyIndexTypes) { - doIndex(null, false); + doIndex(null, Size.REGULAR); } public void index(String login) { requireNonNull(login); - doIndex(login, true); + doIndex(login, Size.LARGE); } - private void doIndex(@Nullable String login, boolean largeBulkIndexing) { + private void doIndex(@Nullable String login, Size bulkSize) { final BulkIndexer bulk = new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex()); - bulk.setLarge(largeBulkIndexing); + bulk.setSize(bulkSize); try (DbSession dbSession = dbClient.openSession(false)) { try (UserResultSetIterator rowIt = UserResultSetIterator.create(dbClient, dbSession, login)) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java index c9059b91430..0f0eed695d2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -29,6 +29,7 @@ import org.sonar.db.DbSession; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.UuidWithProjectUuidDto; import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.StartupIndexer; @@ -58,7 +59,7 @@ public class ViewIndexer implements StartupIndexer { for (UuidWithProjectUuidDto uuidWithProjectUuidDto : dbClient.componentDao().selectAllViewsAndSubViews(dbSession)) { viewAndProjectViewUuidMap.put(uuidWithProjectUuidDto.getUuid(), uuidWithProjectUuidDto.getProjectUuid()); } - index(dbSession, viewAndProjectViewUuidMap, false, true); + index(dbSession, viewAndProjectViewUuidMap, false, Size.LARGE); } } @@ -75,7 +76,7 @@ public class ViewIndexer implements StartupIndexer { for (ComponentDto viewOrSubView : dbClient.componentDao().selectEnabledDescendantModules(dbSession, rootViewUuid)) { viewAndProjectViewUuidMap.put(viewOrSubView.uuid(), viewOrSubView.projectUuid()); } - index(dbSession, viewAndProjectViewUuidMap, true, false); + index(dbSession, viewAndProjectViewUuidMap, true, Size.REGULAR); } finally { dbSession.close(); } @@ -93,9 +94,9 @@ public class ViewIndexer implements StartupIndexer { bulk.stop(); } - private void index(DbSession dbSession, Map<String, String> viewAndProjectViewUuidMap, boolean needClearCache, boolean largeBulkIndexing) { + private void index(DbSession dbSession, Map<String, String> viewAndProjectViewUuidMap, boolean needClearCache, Size bulkSize) { final BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex()); - bulk.setLarge(largeBulkIndexing); + bulk.setSize(bulkSize); bulk.start(); for (Map.Entry<String, String> entry : viewAndProjectViewUuidMap.entrySet()) { String viewUuid = entry.getKey(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java index e06004f2f5d..c56819fe9cf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Rule; import org.junit.Test; +import org.sonar.server.es.BulkIndexer.Size; import static org.assertj.core.api.Assertions.assertThat; import static org.sonar.server.es.FakeIndexDefinition.INDEX; @@ -68,7 +69,7 @@ public class BulkIndexerTest { BulkIndexer indexer = new BulkIndexer(esTester.client(), INDEX) .setFlushByteSize(500) - .setLarge(true); + .setSize(Size.LARGE); indexer.start(); // replicas are temporarily disabled |