From 06443d5ca9bd2894dcb796179f3af09babc1aa0d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lievremont Date: Thu, 4 Dec 2014 16:31:17 +0100 Subject: [PATCH] WIP SONAR-5802 Log progress of bulk index --- .../server/db/migrations/MassUpdate.java | 29 +---------- ...dMissingCustomRuleParametersMigration.java | 8 +-- .../PopulateProjectsUuidColumnsMigration.java | 8 +-- .../java/org/sonar/server/es/BulkIndexer.java | 19 +++++++ .../org/sonar/server/util/ProgressTask.java | 52 +++++++++++++++++++ 5 files changed, 83 insertions(+), 33 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/util/ProgressTask.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java index 845b22c5d84..ee9b0a884a3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java @@ -19,14 +19,13 @@ */ package org.sonar.server.db.migrations; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.core.persistence.Database; +import org.sonar.server.util.ProgressTask; import java.sql.Connection; import java.sql.SQLException; import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.atomic.AtomicLong; public class MassUpdate { @@ -43,7 +42,7 @@ public class MassUpdate { private final Database db; private final Connection readConnection, writeConnection; private final AtomicLong counter = new AtomicLong(0L); - private final ProgressTask progressTask = new ProgressTask(counter); + private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("DbMigration")); private Select select; private Upsert update; @@ -99,28 +98,4 @@ public class MassUpdate { } } - public static class ProgressTask extends TimerTask { - private static final Logger LOGGER = LoggerFactory.getLogger("DbMigration"); - public static final long PERIOD_MS = 60000L; - private final AtomicLong counter; - private String rowName = "rows"; - - public ProgressTask(AtomicLong counter) { - this.counter = counter; - } - - void setRowPluralName(String s) { - this.rowName = s; - } - - @Override - public void run() { - log(); - } - - public void log() { - LOGGER.info(String.format("%d %s processed", counter.get(), rowName)); - } - } - } diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java index c58f3f6ae47..0b82179025e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java @@ -24,6 +24,8 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashMultimap; import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; + +import org.slf4j.LoggerFactory; import org.sonar.api.utils.System2; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.migration.v45.Migration45Mapper; @@ -31,7 +33,7 @@ import org.sonar.core.persistence.migration.v45.Rule; import org.sonar.core.persistence.migration.v45.RuleParameter; import org.sonar.server.db.DbClient; import org.sonar.server.db.migrations.DatabaseMigration; -import org.sonar.server.db.migrations.MassUpdate; +import org.sonar.server.util.ProgressTask; import javax.annotation.Nullable; @@ -54,7 +56,7 @@ public class AddMissingCustomRuleParametersMigration implements DatabaseMigratio private final System2 system; private final AtomicLong counter = new AtomicLong(0L); - private final MassUpdate.ProgressTask progressTask = new MassUpdate.ProgressTask(counter); + private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("DbMigration")); public AddMissingCustomRuleParametersMigration(DbClient db, System2 system) { this.db = db; @@ -64,7 +66,7 @@ public class AddMissingCustomRuleParametersMigration implements DatabaseMigratio @Override public void execute() { Timer timer = new Timer("Db Migration Progress"); - timer.schedule(progressTask, MassUpdate.ProgressTask.PERIOD_MS, MassUpdate.ProgressTask.PERIOD_MS); + timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS); DbSession session = db.openSession(false); try { diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java index 1e6cab9ba16..5c7323d8c15 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java @@ -22,8 +22,10 @@ package org.sonar.server.db.migrations.v50; import com.google.common.base.Splitter; import com.google.common.base.Strings; + import org.apache.ibatis.session.ResultContext; import org.apache.ibatis.session.ResultHandler; +import org.slf4j.LoggerFactory; import org.sonar.api.resources.Scopes; import org.sonar.api.utils.internal.Uuids; import org.sonar.core.persistence.DbSession; @@ -31,7 +33,7 @@ import org.sonar.core.persistence.migration.v50.Component; import org.sonar.core.persistence.migration.v50.Migration50Mapper; import org.sonar.server.db.DbClient; import org.sonar.server.db.migrations.DatabaseMigration; -import org.sonar.server.db.migrations.MassUpdate; +import org.sonar.server.util.ProgressTask; import java.util.List; import java.util.Map; @@ -50,7 +52,7 @@ public class PopulateProjectsUuidColumnsMigration implements DatabaseMigration { private final DbClient db; private final AtomicLong counter = new AtomicLong(0L); - private final MassUpdate.ProgressTask progressTask = new MassUpdate.ProgressTask(counter); + private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("DbMigration")); public PopulateProjectsUuidColumnsMigration(DbClient db) { this.db = db; @@ -59,7 +61,7 @@ public class PopulateProjectsUuidColumnsMigration implements DatabaseMigration { @Override public void execute() { Timer timer = new Timer("Db Migration Progress"); - timer.schedule(progressTask, MassUpdate.ProgressTask.PERIOD_MS, MassUpdate.ProgressTask.PERIOD_MS); + timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS); final DbSession readSession = db.openSession(false); final DbSession writeSession = db.openSession(true); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index 972f7269b28..5961a99ca0f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -21,6 +21,7 @@ package org.sonar.server.es; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; + import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; @@ -29,8 +30,13 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.picocontainer.Startable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.server.util.ProgressTask; import java.util.Map; +import java.util.Timer; +import java.util.concurrent.atomic.AtomicLong; /** * Helper to bulk requests in an efficient way : @@ -53,6 +59,10 @@ public class BulkIndexer implements Startable { private BulkRequestBuilder bulkRequest = null; private Map largeInitialSettings = null; + private final AtomicLong counter = new AtomicLong(0L); + private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("BulkIndex")).setRowPluralName("requests"); + private final Timer timer = new Timer("Bulk index progress"); + public BulkIndexer(EsClient client, String indexName) { this.client = client; this.indexName = indexName; @@ -107,10 +117,12 @@ public class BulkIndexer implements Startable { updateSettings(bulkSettings); } bulkRequest = client.prepareBulk(); + timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS); } public void add(ActionRequest request) { bulkRequest.request().add(request); + counter.getAndIncrement(); if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) { executeBulk(bulkRequest); bulkRequest = client.prepareBulk(); @@ -122,6 +134,13 @@ public class BulkIndexer implements Startable { if (bulkRequest.numberOfActions() > 0) { executeBulk(bulkRequest); } + + // Log final advancement and reset counter + progressTask.log(); + counter.set(0L); + timer.cancel(); + timer.purge(); + if (refresh) { client.prepareRefresh(indexName).get(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/util/ProgressTask.java b/server/sonar-server/src/main/java/org/sonar/server/util/ProgressTask.java new file mode 100644 index 00000000000..99b9d197c56 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/util/ProgressTask.java @@ -0,0 +1,52 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.util; + +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ProgressTask extends TimerTask { + private final Logger logger; + public static final long PERIOD_MS = 60000L; + private final AtomicLong counter; + private String rowName = "rows"; + + public ProgressTask(AtomicLong counter, Logger logger) { + this.counter = counter; + this.logger = logger; + } + + public ProgressTask setRowPluralName(String s) { + this.rowName = s; + return this; + } + + @Override + public void run() { + log(); + } + + public void log() { + logger.info(String.format("%d %s processed", counter.get(), rowName)); + } +} -- 2.39.5