WIP SONAR-5802 Log progress of bulk index

5 files changed, 83 insertions, 33 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java
index 845b22c5d84..ee9b0a884a3 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/MassUpdate.java
@@ -19,14 +19,13 @@
  */
 package org.sonar.server.db.migrations;
 
-import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.core.persistence.Database;
+import org.sonar.server.util.ProgressTask;
 
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.Timer;
-import java.util.TimerTask;
 import java.util.concurrent.atomic.AtomicLong;
 
 public class MassUpdate {
@@ -43,7 +42,7 @@ public class MassUpdate {
   private final Database db;
   private final Connection readConnection, writeConnection;
   private final AtomicLong counter = new AtomicLong(0L);
-  private final ProgressTask progressTask = new ProgressTask(counter);
+  private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("DbMigration"));
 
   private Select select;
   private Upsert update;
@@ -99,28 +98,4 @@ public class MassUpdate {
     }
   }
 
-  public static class ProgressTask extends TimerTask {
-    private static final Logger LOGGER = LoggerFactory.getLogger("DbMigration");
-    public static final long PERIOD_MS = 60000L;
-    private final AtomicLong counter;
-    private String rowName = "rows";
-
-    public ProgressTask(AtomicLong counter) {
-      this.counter = counter;
-    }
-
-    void setRowPluralName(String s) {
-      this.rowName = s;
-    }
-
-    @Override
-    public void run() {
-      log();
-    }
-
-    public void log() {
-      LOGGER.info(String.format("%d %s processed", counter.get(), rowName));
-    }
-  }
-
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java
index c58f3f6ae47..0b82179025e 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v451/AddMissingCustomRuleParametersMigration.java
@@ -24,6 +24,8 @@ import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Multimap;
+
+import org.slf4j.LoggerFactory;
 import org.sonar.api.utils.System2;
 import org.sonar.core.persistence.DbSession;
 import org.sonar.core.persistence.migration.v45.Migration45Mapper;
@@ -31,7 +33,7 @@ import org.sonar.core.persistence.migration.v45.Rule;
 import org.sonar.core.persistence.migration.v45.RuleParameter;
 import org.sonar.server.db.DbClient;
 import org.sonar.server.db.migrations.DatabaseMigration;
-import org.sonar.server.db.migrations.MassUpdate;
+import org.sonar.server.util.ProgressTask;
 
 import javax.annotation.Nullable;
 
@@ -54,7 +56,7 @@ public class AddMissingCustomRuleParametersMigration implements DatabaseMigratio
   private final System2 system;
 
   private final AtomicLong counter = new AtomicLong(0L);
-  private final MassUpdate.ProgressTask progressTask = new MassUpdate.ProgressTask(counter);
+  private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("DbMigration"));
 
   public AddMissingCustomRuleParametersMigration(DbClient db, System2 system) {
     this.db = db;
@@ -64,7 +66,7 @@ public class AddMissingCustomRuleParametersMigration implements DatabaseMigratio
   @Override
   public void execute() {
     Timer timer = new Timer("Db Migration Progress");
-    timer.schedule(progressTask, MassUpdate.ProgressTask.PERIOD_MS, MassUpdate.ProgressTask.PERIOD_MS);
+    timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
 
     DbSession session = db.openSession(false);
     try {
diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java
index 1e6cab9ba16..5c7323d8c15 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/PopulateProjectsUuidColumnsMigration.java
@@ -22,8 +22,10 @@ package org.sonar.server.db.migrations.v50;
 
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
+
 import org.apache.ibatis.session.ResultContext;
 import org.apache.ibatis.session.ResultHandler;
+import org.slf4j.LoggerFactory;
 import org.sonar.api.resources.Scopes;
 import org.sonar.api.utils.internal.Uuids;
 import org.sonar.core.persistence.DbSession;
@@ -31,7 +33,7 @@ import org.sonar.core.persistence.migration.v50.Component;
 import org.sonar.core.persistence.migration.v50.Migration50Mapper;
 import org.sonar.server.db.DbClient;
 import org.sonar.server.db.migrations.DatabaseMigration;
-import org.sonar.server.db.migrations.MassUpdate;
+import org.sonar.server.util.ProgressTask;
 
 import java.util.List;
 import java.util.Map;
@@ -50,7 +52,7 @@ public class PopulateProjectsUuidColumnsMigration implements DatabaseMigration {
 
   private final DbClient db;
   private final AtomicLong counter = new AtomicLong(0L);
-  private final MassUpdate.ProgressTask progressTask = new MassUpdate.ProgressTask(counter);
+  private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("DbMigration"));
 
   public PopulateProjectsUuidColumnsMigration(DbClient db) {
     this.db = db;
@@ -59,7 +61,7 @@ public class PopulateProjectsUuidColumnsMigration implements DatabaseMigration {
   @Override
   public void execute() {
     Timer timer = new Timer("Db Migration Progress");
-    timer.schedule(progressTask, MassUpdate.ProgressTask.PERIOD_MS, MassUpdate.ProgressTask.PERIOD_MS);
+    timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
 
     final DbSession readSession = db.openSession(false);
     final DbSession writeSession = db.openSession(true);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
index 972f7269b28..5961a99ca0f 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
@@ -21,6 +21,7 @@ package org.sonar.server.es;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
+
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
@@ -29,8 +30,13 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.picocontainer.Startable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.server.util.ProgressTask;
 
 import java.util.Map;
+import java.util.Timer;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Helper to bulk requests in an efficient way :
@@ -53,6 +59,10 @@ public class BulkIndexer implements Startable {
   private BulkRequestBuilder bulkRequest = null;
   private Map<String, Object> largeInitialSettings = null;
 
+  private final AtomicLong counter = new AtomicLong(0L);
+  private final ProgressTask progressTask = new ProgressTask(counter, LoggerFactory.getLogger("BulkIndex")).setRowPluralName("requests");
+  private final Timer timer = new Timer("Bulk index progress");
+
   public BulkIndexer(EsClient client, String indexName) {
     this.client = client;
     this.indexName = indexName;
@@ -107,10 +117,12 @@ public class BulkIndexer implements Startable {
       updateSettings(bulkSettings);
     }
     bulkRequest = client.prepareBulk();
+    timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
   }
 
   public void add(ActionRequest request) {
     bulkRequest.request().add(request);
+    counter.getAndIncrement();
     if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) {
       executeBulk(bulkRequest);
       bulkRequest = client.prepareBulk();
@@ -122,6 +134,13 @@ public class BulkIndexer implements Startable {
     if (bulkRequest.numberOfActions() > 0) {
       executeBulk(bulkRequest);
     }
+
+    // Log final advancement and reset counter
+    progressTask.log();
+    counter.set(0L);
+    timer.cancel();
+    timer.purge();
+
     if (refresh) {
       client.prepareRefresh(indexName).get();
     }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/util/ProgressTask.java b/server/sonar-server/src/main/java/org/sonar/server/util/ProgressTask.java
new file mode 100644
index 00000000000..99b9d197c56
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/util/ProgressTask.java
@@ -0,0 +1,52 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.util;
+
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProgressTask extends TimerTask {
+  private final Logger logger;
+  public static final long PERIOD_MS = 60000L;
+  private final AtomicLong counter;
+  private String rowName = "rows";
+
+  public ProgressTask(AtomicLong counter, Logger logger) {
+    this.counter = counter;
+    this.logger = logger;
+  }
+
+  public ProgressTask setRowPluralName(String s) {
+    this.rowName = s;
+    return this;
+  }
+
+  @Override
+  public void run() {
+    log();
+  }
+
+  public void log() {
+    logger.info(String.format("%d %s processed", counter.get(), rowName));
+  }
+}