SONAR-4691 do not extensively use Oracle rollback segments

16 files changed, 355 insertions, 189 deletions

diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/TestDatabase.java b/sonar-core/src/test/java/org/sonar/core/persistence/TestDatabase.java
index 796625ca1be..63a4c63798a 100644
--- a/sonar-core/src/test/java/org/sonar/core/persistence/TestDatabase.java
+++ b/sonar-core/src/test/java/org/sonar/core/persistence/TestDatabase.java
@@ -55,6 +55,8 @@ import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
@@ -143,6 +145,28 @@ public class TestDatabase extends ExternalResource {
     }
   }
 
+  public int count(String sql) {
+    Connection connection = null;
+    PreparedStatement stmt = null;
+      ResultSet rs = null;
+    try {
+      connection = openConnection();
+      stmt = connection.prepareStatement(sql);
+      rs = stmt.executeQuery();
+      if (rs.next()) {
+        return rs.getInt(1);
+      }
+      throw new IllegalStateException("No results for " + sql);
+
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to execute sql: " + sql);
+
+    } finally {
+      DbUtils.closeQuietly(connection, stmt, rs);
+    }
+  }
+
+
   public void prepareDbUnit(Class testClass, String... testNames) {
     InputStream[] streams = new InputStream[testNames.length];
     try {
@@ -278,5 +302,4 @@ public class TestDatabase extends ExternalResource {
       IOUtils.closeQuietly(input);
     }
   }
-
 }
diff --git a/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigration.java b/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigration.java
index 2e99e4b30bb..f49d40d0505 100644
--- a/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigration.java
+++ b/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigration.java
@@ -19,14 +19,12 @@
  */
 package org.sonar.server.db;
 
-import org.sonar.core.persistence.Database;
-
 /**
- * Java alternative of ActiveRecord::Migration.
+ * Java alternative of ActiveRecord::Migration. Do not forget to declare implementation classes in {@link DatabaseMigrations#CLASSES}
  * @since 3.7
  */
 public interface DatabaseMigration {
 
-  void execute(Database db);
+  void execute();
 
 }
diff --git a/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigrations.java b/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigrations.java
new file mode 100644
index 00000000000..23e4a8a52b0
--- /dev/null
+++ b/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigrations.java
@@ -0,0 +1,30 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.db;
+
+import org.sonar.server.db.DatabaseMigration;
+import org.sonar.server.db.migrations.violation.ViolationMigration;
+
+public interface DatabaseMigrations {
+
+  Class<? extends DatabaseMigration>[] CLASSES = new Class[]{
+    ViolationMigration.class
+  };
+}
diff --git a/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigrator.java b/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigrator.java
index 6b36e055134..c8012b3ca65 100644
--- a/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigrator.java
+++ b/sonar-server/src/main/java/org/sonar/server/db/DatabaseMigrator.java
@@ -40,10 +40,12 @@ public class DatabaseMigrator implements ServerComponent {
 
   private final MyBatis myBatis;
   private final Database database;
+  private final DatabaseMigration[] migrations;
 
-  public DatabaseMigrator(MyBatis myBatis, Database database) {
+  public DatabaseMigrator(MyBatis myBatis, Database database, DatabaseMigration[] migrations) {
     this.myBatis = myBatis;
     this.database = database;
+    this.migrations = migrations;
   }
 
   /**
@@ -72,10 +74,9 @@ public class DatabaseMigrator implements ServerComponent {
   }
 
   public void executeMigration(String className) {
+    DatabaseMigration migration = getMigration(className);
     try {
-      Class<DatabaseMigration> migrationClass = (Class<DatabaseMigration>) Class.forName(className);
-      DatabaseMigration migration = migrationClass.newInstance();
-      migration.execute(database);
+      migration.execute();
 
     } catch (Exception e) {
       // duplication between log and exception because webapp does not correctly log initial stacktrace
@@ -85,6 +86,15 @@ public class DatabaseMigrator implements ServerComponent {
     }
   }
 
+  private DatabaseMigration getMigration(String className) {
+    for (DatabaseMigration migration : migrations) {
+      if (migration.getClass().getName().equals(className)) {
+        return migration;
+      }
+    }
+    throw new IllegalArgumentException("Database migration not found: " + className);
+  }
+
   @VisibleForTesting
   protected void createSchema(Connection connection, String dialectId) {
     DdlUtils.createSchema(connection, dialectId);
diff --git a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Progress.java b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Progress.java
index fe1175a8d7b..8057855c913 100644
--- a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Progress.java
+++ b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Progress.java
@@ -52,14 +52,13 @@ class Progress extends TimerTask {
   public void run() {
     int totalIssues = counter.get();
     long durationMinutes = (System.currentTimeMillis() - start) / 60000L;
-    int frequency = 0, remaining = 0;
+    int remaining = 0;
     if (durationMinutes > 0) {
-      frequency = (int) (totalIssues / durationMinutes);
+      int frequency = (int) (totalIssues / durationMinutes);
       remaining = (totalViolations - totalIssues) / frequency;
     }
     logger.info(String.format(
-      "%d%% [%d/%d violations, %d violations/minute, %d minutes remaining]",
-      (100 * totalIssues) / totalViolations, totalIssues, totalViolations, frequency, remaining)
+      "%d%% [%d/%d violations, %d minutes remaining]", (100 * totalIssues) / totalViolations, totalIssues, totalViolations, remaining)
     );
   }
 }
diff --git a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Referentials.java b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Referentials.java
index d4d28258488..35ac7c60c81 100644
--- a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Referentials.java
+++ b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/Referentials.java
@@ -32,18 +32,24 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 class Referentials {
-  private final Database database;
+
+  static final int VIOLATION_GROUP_SIZE = 1000;
+
   private final Map<Long, String> loginsByUserId;
   private final Map<Long, String> plansById;
-  private final int totalViolations;
+  private final Queue<long[]> groupsOfViolationIds;
+
+  // int is enough, it allows to upgrade up to 2 billions violations !
+  private int totalViolations = 0;
 
   Referentials(Database database) throws SQLException {
-    this.database = database;
-    loginsByUserId = selectLongString("select id,login from users");
-    plansById = selectLongString("select id,kee from action_plans");
-    totalViolations = selectInt("select count(id) from rule_failures");
+    loginsByUserId = selectLongString(database, "select id,login from users");
+    plansById = selectLongString(database, "select id,kee from action_plans");
+    groupsOfViolationIds = initGroupOfViolationIds(database);
   }
 
   @CheckForNull
@@ -60,7 +66,20 @@ class Referentials {
     return totalViolations;
   }
 
-  private Map<Long, String> selectLongString(String sql) throws SQLException {
+  @CheckForNull
+  Long[] pollGroupOfViolationIds() {
+    long[] longs = groupsOfViolationIds.poll();
+    if (longs == null) {
+      return null;
+    }
+    Long[] objects = new Long[longs.length];
+    for (int i = 0; i < longs.length; i++) {
+      objects[i] = new Long(longs[i]);
+    }
+    return objects;
+  }
+
+  private Map<Long, String> selectLongString(Database database, String sql) throws SQLException {
     Connection connection = database.getDataSource().getConnection();
     try {
       return new QueryRunner().query(connection, sql, new ResultSetHandler<Map<Long, String>>() {
@@ -78,17 +97,33 @@ class Referentials {
     }
   }
 
-  private int selectInt(String sql) throws SQLException {
+  private Queue<long[]> initGroupOfViolationIds(Database database) throws SQLException {
     Connection connection = database.getDataSource().getConnection();
+    connection.setAutoCommit(false);
     Statement stmt = null;
     ResultSet rs = null;
     try {
       stmt = connection.createStatement();
-      rs = stmt.executeQuery(sql);
-      if (rs.next()) {
-        return rs.getInt(1);
+      stmt.setFetchSize(10000);
+      rs = stmt.executeQuery("select id from rule_failures");
+      ConcurrentLinkedQueue<long[]> queue = new ConcurrentLinkedQueue<long[]>();
+
+      totalViolations = 0;
+      long[] block = new long[VIOLATION_GROUP_SIZE];
+      int cursor = 0;
+      while (rs.next()) {
+        block[cursor++] = rs.getLong(1);
+        totalViolations++;
+        if (cursor == VIOLATION_GROUP_SIZE) {
+          queue.add(block);
+          block = new long[VIOLATION_GROUP_SIZE];
+          cursor = 0;
+        }
+      }
+      if (cursor > 0) {
+        queue.add(block);
       }
-      return 0;
+      return queue;
     } finally {
       DbUtils.closeQuietly(connection, stmt, rs);
     }
diff --git a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverter.java b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverter.java
index 1d57602224c..6c819e69913 100644
--- a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverter.java
+++ b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverter.java
@@ -36,8 +36,9 @@ import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 
-class ViolationConverter implements Runnable {
+class ViolationConverter implements Callable<Object> {
 
   private static final long ONE_YEAR = 365L * 24 * 60 * 60 * 1000;
   private static final Date ONE_YEAR_AGO = new Date(System.currentTimeMillis() - ONE_YEAR);
@@ -70,6 +71,7 @@ class ViolationConverter implements Runnable {
   private static final String USER_ID = "userId";
   private static final String SEVERITY_MAJOR = "MAJOR";
 
+
   private static final String SQL_ISSUE_COLUMNS = "kee, component_id, root_component_id, rule_id, severity, manual_severity, message, line, effort_to_fix, status, resolution, " +
     "checksum, reporter, assignee, action_plan_key, issue_attributes, issue_creation_date, issue_update_date, created_at, updated_at";
 
@@ -79,40 +81,78 @@ class ViolationConverter implements Runnable {
   private static final String SQL_INSERT_ISSUE_CHANGE = "INSERT INTO issue_changes(kee, issue_key, user_login, change_type, change_data, created_at, updated_at)" +
     " VALUES (?, ?, ?, 'comment', ?, ?, ?)";
 
+  private static final String SQL_DELETE_RULE_FAILURES;
+
+  static {
+    StringBuilder sb = new StringBuilder("delete rule_failures where ");
+    for (int i = 0; i < Referentials.VIOLATION_GROUP_SIZE; i++) {
+      if (i > 0) {
+        sb.append(" or ");
+      }
+      sb.append("id=?");
+    }
+    SQL_DELETE_RULE_FAILURES = sb.toString();
+  }
+
+  static final String SQL_SELECT_RULE_FAILURES;
+
+  static {
+    StringBuilder sb = new StringBuilder("select rev.id as reviewId, s.project_id as projectId, rf.rule_id as ruleId, " +
+      "  rf.failure_level as failureLevel, rf.message as message, rf.line as line, " +
+      "  rf.cost as cost, rf.created_at as createdAt, rf.checksum as checksum, rev.user_id as reviewReporterId, " +
+      "  rev.assignee_id as reviewAssigneeId, rev.status as reviewStatus, " +
+      "  rev.severity as reviewSeverity, rev.resolution as reviewResolution, rev.manual_severity as reviewManualSeverity, " +
+      "  rev.data as reviewData, rev.updated_at as reviewUpdatedAt, " +
+      "  s.root_project_id as rootProjectId, rev.manual_violation as reviewManualViolation, planreviews.action_plan_id as planId " +
+      " from rule_failures rf " +
+      " inner join snapshots s on s.id=rf.snapshot_id " +
+      " left join reviews rev on rev.rule_failure_permanent_id=rf.permanent_id " +
+      " left join action_plans_reviews planreviews on planreviews.review_id=rev.id " +
+      " where ");
+    for (int i = 0; i < Referentials.VIOLATION_GROUP_SIZE; i++) {
+      if (i > 0) {
+        sb.append(" or ");
+      }
+      sb.append("rf.id=?");
+    }
+    SQL_SELECT_RULE_FAILURES = sb.toString();
+  }
+
   private final Database db;
-  private final Object[] violationIds;
   private final Referentials referentials;
   private final Progress progress;
 
-  ViolationConverter(Referentials referentials, Database db, Object[] violationIds, Progress progress) {
+  ViolationConverter(Referentials referentials, Database db, Progress progress) {
     this.referentials = referentials;
     this.db = db;
-    this.violationIds = violationIds;
     this.progress = progress;
   }
 
   @Override
-  public void run() {
-    convert(selectRows());
+  public Object call() throws Exception {
+    Long[] violationIds = referentials.pollGroupOfViolationIds();
+    while (violationIds != null) {
+      List<Map<String, Object>> rows = selectRows(violationIds);
+      convert(rows, violationIds);
+
+      violationIds = referentials.pollGroupOfViolationIds();
+    }
+    return null;
   }
 
-  private List<Map<String, Object>> selectRows() {
+  private List<Map<String, Object>> selectRows(Long[] violationIds) throws SQLException {
     Connection readConnection = null;
     try {
       readConnection = db.getDataSource().getConnection();
       ViolationHandler violationHandler = new ViolationHandler();
-      return new QueryRunner().query(readConnection, violationHandler.SQL, violationHandler, violationIds);
-
-    } catch (SQLException e) {
-      //TODO
-      throw new IllegalStateException();
+      return new QueryRunner().query(readConnection, SQL_SELECT_RULE_FAILURES, violationHandler, violationIds);
 
     } finally {
       DbUtils.closeQuietly(readConnection);
     }
   }
 
-  private void convert(List<Map<String, Object>> rows) {
+  private void convert(List<Map<String, Object>> rows, Long[] violationIds) throws SQLException {
     Connection readConnection = null;
     Connection writeConnection = null;
     try {
@@ -123,6 +163,7 @@ class ViolationConverter implements Runnable {
       List<Object[]> allParams = Lists.newArrayList();
       List<Map<String, Object>> allComments = Lists.newArrayList();
 
+      QueryRunner runner = new QueryRunner();
       for (Map<String, Object> row : rows) {
         Long componentId = (Long) row.get(PROJECT_ID);
         if (componentId == null) {
@@ -151,7 +192,7 @@ class ViolationConverter implements Runnable {
             reporter = referentials.userLogin((Long) row.get(REVIEW_REPORTER_ID));
           }
 
-          List<Map<String, Object>> comments = new QueryRunner().query(readConnection, ReviewCommentsHandler.SQL + reviewId, new ReviewCommentsHandler());
+          List<Map<String, Object>> comments = runner.query(readConnection, ReviewCommentsHandler.SQL + reviewId, new ReviewCommentsHandler());
           for (Map<String, Object> comment : comments) {
             comment.put(ISSUE_KEY, issueKey);
             allComments.add(comment);
@@ -180,15 +221,12 @@ class ViolationConverter implements Runnable {
         params[19] = updatedAt;
         allParams.add(params);
       }
-      new QueryRunner().batch(writeConnection, SQL_INSERT_ISSUE, allParams.toArray(new Object[allParams.size()][]));
-      writeConnection.commit();
-
+      runner.batch(writeConnection, SQL_INSERT_ISSUE, allParams.toArray(new Object[allParams.size()][]));
       insertComments(writeConnection, allComments);
-      progress.increment(rows.size());
+      runner.update(writeConnection, SQL_DELETE_RULE_FAILURES, violationIds);
 
-    } catch (SQLException e) {
-      //TODO
-      throw new IllegalStateException();
+      writeConnection.commit();
+      progress.increment(rows.size());
 
     } finally {
       DbUtils.closeQuietly(readConnection);
@@ -214,7 +252,6 @@ class ViolationConverter implements Runnable {
     }
     if (!allParams.isEmpty()) {
       new QueryRunner().batch(writeConnection, SQL_INSERT_ISSUE_CHANGE, allParams.toArray(new Object[allParams.size()][]));
-      writeConnection.commit();
     }
   }
 
@@ -236,29 +273,6 @@ class ViolationConverter implements Runnable {
   private static class ViolationHandler extends AbstractListHandler<Map<String, Object>> {
     private static final Map<Integer, String> SEVERITIES = ImmutableMap.of(1, Severity.INFO, 2, Severity.MINOR, 3, Severity.MAJOR, 4, Severity.CRITICAL, 5, Severity.BLOCKER);
 
-    static final String SQL;
-
-    static {
-      StringBuilder sb = new StringBuilder("select rev.id as reviewId, s.project_id as projectId, rf.rule_id as ruleId, " +
-        "  rf.failure_level as failureLevel, rf.message as message, rf.line as line, " +
-        "  rf.cost as cost, rf.created_at as createdAt, rf.checksum as checksum, rev.user_id as reviewReporterId, " +
-        "  rev.assignee_id as reviewAssigneeId, rev.status as reviewStatus, " +
-        "  rev.severity as reviewSeverity, rev.resolution as reviewResolution, rev.manual_severity as reviewManualSeverity, " +
-        "  rev.data as reviewData, rev.updated_at as reviewUpdatedAt, " +
-        "  s.root_project_id as rootProjectId, rev.manual_violation as reviewManualViolation, planreviews.action_plan_id as planId " +
-        " from rule_failures rf " +
-        " inner join snapshots s on s.id=rf.snapshot_id " +
-        " left join reviews rev on rev.rule_failure_permanent_id=rf.permanent_id " +
-        " left join action_plans_reviews planreviews on planreviews.review_id=rev.id " +
-        " where ");
-      for (int i = 0; i < ViolationMigration.GROUP_SIZE; i++) {
-        if (i > 0) {
-          sb.append(" or ");
-        }
-        sb.append("rf.id=?");
-      }
-      SQL = sb.toString();
-    }
 
     @Override
     protected Map<String, Object> handleRow(ResultSet rs) throws SQLException {
diff --git a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverters.java b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverters.java
index a643fa7fdc9..492d95a6228 100644
--- a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverters.java
+++ b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationConverters.java
@@ -19,41 +19,54 @@
  */
 package org.sonar.server.db.migrations.violation;
 
+import com.google.common.collect.Lists;
+import org.sonar.api.config.Settings;
 import org.sonar.core.persistence.Database;
 
+import java.util.List;
 import java.util.Timer;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 class ViolationConverters {
 
-  static final int MAX_THREADS = 5;
+  static final int DEFAULT_THREADS = 5;
+  static final String THREADS_PROPERTY = "sonar.violationMigration.threads";
+  private final Settings settings;
 
-  private final ExecutorService executorService;
-  private final Database database;
-  private final Referentials referentials;
-  private final Progress progress;
-  private final Timer timer;
-
-  ViolationConverters(Database db, Referentials referentials) {
-    this.database = db;
-    this.referentials = referentials;
+  ViolationConverters(Settings settings) {
+    this.settings = settings;
+  }
 
-    this.progress = new Progress(referentials.totalViolations());
-    timer = new Timer(Progress.THREAD_NAME);
+  void execute(Referentials referentials, Database db) throws Exception {
+    Progress progress = new Progress(referentials.totalViolations());
+    Timer timer = new Timer(Progress.THREAD_NAME);
     timer.schedule(progress, Progress.DELAY_MS, Progress.DELAY_MS);
 
-    BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(MAX_THREADS);
-    RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
-    this.executorService = new ThreadPoolExecutor(0, MAX_THREADS, 5L, TimeUnit.SECONDS, blockingQueue, rejectedExecutionHandler);
-  }
+    List<Callable<Object>> converters = Lists.newArrayList();
+    for (int i = 0; i < numberOfThreads(); i++) {
+      converters.add(new ViolationConverter(referentials, db, progress));
+    }
+    ExecutorService executor = Executors.newFixedThreadPool(converters.size());
+    List<Future<Object>> results = executor.invokeAll(converters);
 
-  void convert(Object[] violationIds) {
-    executorService.execute(new ViolationConverter(referentials, database, violationIds, progress));
+    executor.shutdown();
+    for (Future result : results) {
+      result.get();
+    }
+    timer.cancel();
   }
 
-  void waitForFinished() throws InterruptedException {
-    executorService.shutdown();
-    executorService.awaitTermination(10L, TimeUnit.SECONDS);
-    timer.cancel();
+  int numberOfThreads() {
+    int threads = settings.getInt(THREADS_PROPERTY);
+    if (threads < 0) {
+      throw new IllegalArgumentException(String.format("Bad value of %s: %d", THREADS_PROPERTY, threads));
+    }
+    if (threads == 0) {
+      threads = DEFAULT_THREADS;
+    }
+    return threads;
   }
 }
diff --git a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationMigration.java b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationMigration.java
index 86a94ddef0c..8dae2f15034 100644
--- a/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationMigration.java
+++ b/sonar-server/src/main/java/org/sonar/server/db/migrations/violation/ViolationMigration.java
@@ -19,17 +19,13 @@
  */
 package org.sonar.server.db.migrations.violation;
 
-import org.apache.commons.dbutils.DbUtils;
-import org.apache.commons.dbutils.QueryRunner;
-import org.apache.commons.dbutils.ResultSetHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.api.config.Settings;
 import org.sonar.api.utils.MessageException;
 import org.sonar.core.persistence.Database;
 import org.sonar.server.db.DatabaseMigration;
 
-import java.sql.Connection;
-import java.sql.ResultSet;
 import java.sql.SQLException;
 
 /**
@@ -37,15 +33,21 @@ import java.sql.SQLException;
  */
 public class ViolationMigration implements DatabaseMigration {
 
-  public static final int GROUP_SIZE = 1000;
+
+  private final Settings settings;
 
   private Logger logger = LoggerFactory.getLogger(ViolationMigration.class);
+  private final Database db;
+
+  public ViolationMigration(Database database, Settings settings) {
+    this.db = database;
+    this.settings = settings;
+  }
 
   @Override
-  public void execute(Database db) {
+  public void execute() {
     try {
-      truncateIssueTables(db);
-      migrate(db);
+      migrate();
 
     } catch (SQLException e) {
       logger.error("Fail to convert violations to issues", e);
@@ -58,72 +60,17 @@ public class ViolationMigration implements DatabaseMigration {
     }
   }
 
-  private void truncateIssueTables(Database db) throws SQLException {
-    Connection connection = null;
-    try {
-      QueryRunner runner = new QueryRunner();
-      connection = db.getDataSource().getConnection();
-      connection.setAutoCommit(true);
-
-      // lower-case table names for SQLServer....
-      runner.update(connection, "TRUNCATE TABLE issues");
-      runner.update(connection, "TRUNCATE TABLE issue_changes");
-
-    } finally {
-      DbUtils.closeQuietly(connection);
-    }
-
-  }
-
-  public void migrate(Database db) throws Exception {
+  public void migrate() throws Exception {
+    logger.info("Initialize input");
     Referentials referentials = new Referentials(db);
-    if (referentials.totalViolations() > 0) {
-      ViolationConverters converters = new ViolationConverters(db, referentials);
-      Connection readConnection = db.getDataSource().getConnection();
-      try {
-        new QueryRunner().query(readConnection, "select id from rule_failures", new ViolationIdHandler(converters));
-      } finally {
-        DbUtils.closeQuietly(readConnection);
-      }
-      converters.waitForFinished();
-    }
-  }
-
-  private static class ViolationIdHandler implements ResultSetHandler {
-    private final ViolationConverters converters;
 
-    private ViolationIdHandler(ViolationConverters converters) {
-      this.converters = converters;
-    }
-
-    @Override
-    public Object handle(ResultSet rs) throws SQLException {
-      // int is enough, it allows to upgrade up to 2 billions violations !
-      int total = 0;
-      int cursor = 0;
+    if (referentials.totalViolations() > 0) {
+      logger.info("Migrate {} violations", referentials.totalViolations());
 
-      Object[] violationIds = new Object[GROUP_SIZE];
-      while (rs.next()) {
-        long violationId = rs.getLong(1);
-        violationIds[cursor++] = violationId;
-        if (cursor == GROUP_SIZE) {
-          converters.convert(violationIds);
-          violationIds = new Object[GROUP_SIZE];
-          cursor = 0;
-        }
-        total++;
-      }
-      if (cursor > 0) {
-        for (int i = 0; i < violationIds.length; i++) {
-          if (violationIds[i] == null) {
-            violationIds[i] = -1;
-          }
-        }
-        converters.convert(violationIds);
-      }
-      LoggerFactory.getLogger(getClass()).info(String.format("%d violations migrated to issues", total));
-      return null;
+      ViolationConverters converters = new ViolationConverters(settings);
+      converters.execute(referentials, db);
     }
   }
 
+
 }
diff --git a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java
index b473ca01d81..44b95743aa8 100644
--- a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java
+++ b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java
@@ -75,8 +75,10 @@ import org.sonar.server.component.DefaultComponentFinder;
 import org.sonar.server.component.DefaultRubyComponentService;
 import org.sonar.server.configuration.Backup;
 import org.sonar.server.configuration.ProfilesManager;
+import org.sonar.server.db.DatabaseMigration;
 import org.sonar.server.db.DatabaseMigrator;
 import org.sonar.server.db.EmbeddedDatabaseFactory;
+import org.sonar.server.db.DatabaseMigrations;
 import org.sonar.server.issue.*;
 import org.sonar.server.notifications.NotificationCenter;
 import org.sonar.server.notifications.NotificationService;
@@ -175,6 +177,9 @@ public final class Platform {
     rootContainer.addSingleton(MyBatis.class);
     rootContainer.addSingleton(DefaultServerUpgradeStatus.class);
     rootContainer.addSingleton(DatabaseServerCompatibility.class);
+    for (Class<? extends DatabaseMigration> migrationClass : DatabaseMigrations.CLASSES) {
+      rootContainer.addSingleton(migrationClass);
+    }
     rootContainer.addSingleton(DatabaseMigrator.class);
     rootContainer.addSingleton(DatabaseVersion.class);
     for (Class daoClass : DaoUtils.getDaoClasses()) {
diff --git a/sonar-server/src/test/java/org/sonar/server/db/DatabaseMigratorTest.java b/sonar-server/src/test/java/org/sonar/server/db/DatabaseMigratorTest.java
index 2cc20b3c5e3..1e86f4a0597 100644
--- a/sonar-server/src/test/java/org/sonar/server/db/DatabaseMigratorTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/db/DatabaseMigratorTest.java
@@ -42,12 +42,13 @@ public class DatabaseMigratorTest extends AbstractDaoTestCase {
 
   MyBatis mybatis = mock(MyBatis.class);
   Database database = mock(Database.class);
+  DatabaseMigration[] migrations = new DatabaseMigration[]{new FakeMigration()};
 
   @Test
   public void should_support_only_creation_of_h2_database() throws Exception {
     when(database.getDialect()).thenReturn(new MySql());
 
-    DatabaseMigrator migrator = new DatabaseMigrator(mybatis, database);
+    DatabaseMigrator migrator = new DatabaseMigrator(mybatis, database, migrations);
 
     assertThat(migrator.createDatabase()).isFalse();
     verifyZeroInteractions(mybatis);
@@ -55,25 +56,16 @@ public class DatabaseMigratorTest extends AbstractDaoTestCase {
 
   @Test
   public void fail_if_execute_unknown_migration() throws Exception {
-    thrown.expect(IllegalStateException.class);
-    thrown.expectMessage("Fail to execute database migration: org.xxx.UnknownMigration");
+    thrown.expect(IllegalArgumentException.class);
+    thrown.expectMessage("Database migration not found: org.xxx.UnknownMigration");
 
-    DatabaseMigrator migrator = new DatabaseMigrator(mybatis, database);
+    DatabaseMigrator migrator = new DatabaseMigrator(mybatis, database, migrations);
     migrator.executeMigration("org.xxx.UnknownMigration");
   }
 
   @Test
-  public void fail_if_execute_not_a_migration() throws Exception {
-    thrown.expect(IllegalStateException.class);
-    thrown.expectMessage("Fail to execute database migration: java.lang.String");
-
-    DatabaseMigrator migrator = new DatabaseMigrator(mybatis, database);
-    migrator.executeMigration("java.lang.String");
-  }
-
-  @Test
   public void execute_migration() throws Exception {
-    DatabaseMigrator migrator = new DatabaseMigrator(mybatis, database);
+    DatabaseMigrator migrator = new DatabaseMigrator(mybatis, database, migrations);
     assertThat(FakeMigration.executed).isFalse();
     migrator.executeMigration(FakeMigration.class.getName());
     assertThat(FakeMigration.executed).isTrue();
@@ -89,7 +81,7 @@ public class DatabaseMigratorTest extends AbstractDaoTestCase {
     when(session.getConnection()).thenReturn(connection);
     when(mybatis.openSession()).thenReturn(session);
 
-    DatabaseMigrator databaseMigrator = new DatabaseMigrator(mybatis, database) {
+    DatabaseMigrator databaseMigrator = new DatabaseMigrator(mybatis, database, migrations) {
       @Override
       protected void createSchema(Connection connection, String dialectId) {
       }
@@ -101,7 +93,7 @@ public class DatabaseMigratorTest extends AbstractDaoTestCase {
   public static class FakeMigration implements DatabaseMigration {
     static boolean executed = false;
     @Override
-    public void execute(Database db) {
+    public void execute() {
       executed = true;
     }
   }
diff --git a/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ProgressTest.java b/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ProgressTest.java
index 1e437c9794d..7e3e69de4c0 100644
--- a/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ProgressTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ProgressTest.java
@@ -43,8 +43,8 @@ public class ProgressTest {
     progress.run();
 
     verify(logger, times(3)).info(argument.capture());
-    assertThat(argument.getAllValues().get(0)).matches("0% \\[0/5000 violations, 0 violations/minute, \\d+ minutes remaining\\]");
-    assertThat(argument.getAllValues().get(1)).matches("6% \\[330/5000 violations, \\d+ violations/minute, \\d+ minutes remaining\\]");
-    assertThat(argument.getAllValues().get(2)).matches("40% \\[2000/5000 violations, \\d+ violations/minute, \\d+ minutes remaining\\]");
+    assertThat(argument.getAllValues().get(0)).matches("0% \\[0/5000 violations, \\d+ minutes remaining\\]");
+    assertThat(argument.getAllValues().get(1)).matches("6% \\[330/5000 violations, \\d+ minutes remaining\\]");
+    assertThat(argument.getAllValues().get(2)).matches("40% \\[2000/5000 violations, \\d+ minutes remaining\\]");
   }
 }
diff --git a/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ViolationConvertersTest.java b/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ViolationConvertersTest.java
new file mode 100644
index 00000000000..f3cf0858c57
--- /dev/null
+++ b/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ViolationConvertersTest.java
@@ -0,0 +1,54 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.db.migrations.violation;
+
+import org.junit.Test;
+import org.sonar.api.config.Settings;
+
+import static org.fest.assertions.Assertions.assertThat;
+import static org.fest.assertions.Fail.fail;
+
+public class ViolationConvertersTest {
+
+  @Test
+  public void default_number_of_threads() throws Exception {
+    assertThat(new ViolationConverters(new Settings()).numberOfThreads()).isEqualTo(ViolationConverters.DEFAULT_THREADS);
+  }
+
+  @Test
+  public void configure_number_of_threads() throws Exception {
+    Settings settings = new Settings();
+    settings.setProperty(ViolationConverters.THREADS_PROPERTY, 2);
+    assertThat(new ViolationConverters(settings).numberOfThreads()).isEqualTo(2);
+  }
+
+  @Test
+  public void number_of_threads_should_not_be_negative() throws Exception {
+    try {
+      Settings settings = new Settings();
+      settings.setProperty(ViolationConverters.THREADS_PROPERTY, -2);
+      new ViolationConverters(settings).numberOfThreads();
+      fail();
+    } catch (IllegalArgumentException e) {
+      assertThat(e.getMessage()).isEqualTo("Bad value of " + ViolationConverters.THREADS_PROPERTY + ": -2");
+    }
+  }
+
+}
diff --git a/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ViolationMigrationTest.java b/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ViolationMigrationTest.java
index 58e96d25e8b..a73daf605c0 100644
--- a/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ViolationMigrationTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/db/migrations/violation/ViolationMigrationTest.java
@@ -21,11 +21,13 @@ package org.sonar.server.db.migrations.violation;
 
 import org.junit.Rule;
 import org.junit.Test;
+import org.sonar.api.config.Settings;
 import org.sonar.core.persistence.TestDatabase;
 
 import java.util.Set;
 
 import static org.fest.assertions.Assertions.assertThat;
+import static org.fest.assertions.Fail.fail;
 
 public class ViolationMigrationTest {
 
@@ -36,9 +38,10 @@ public class ViolationMigrationTest {
   public void migrate_violations() throws Exception {
     db.prepareDbUnit(getClass(), "migrate_violations.xml");
 
-    new ViolationMigration().execute(db.database());
+    new ViolationMigration(db.database(), new Settings()).execute();
 
     db.assertDbUnit(getClass(), "migrate_violations_result.xml", "issues", "issue_changes");
+    assertThat(db.count("select count(id) from rule_failures")).isEqualTo(0);
 
     // Progress thread is dead
     Set<Thread> threads = Thread.getAllStackTraces().keySet();
@@ -46,4 +49,6 @@ public class ViolationMigrationTest {
       assertThat(thread.getName()).isNotEqualTo(Progress.THREAD_NAME);
     }
   }
+
+
 }
diff --git a/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations.xml b/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations.xml
index 5a97e0df76e..0d4a6e8ebc0 100644
--- a/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations.xml
+++ b/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations.xml
@@ -29,13 +29,11 @@
 
 
 
-
-
-  <!-- to be truncated -->
+  <!-- Already there from a past migration that crashed -->
   <issues ID="1" COMPONENT_ID="11" ROOT_COMPONENT_ID="10" RULE_ID="20" SEVERITY="MINOR" KEE="[ignore]"
           ACTION_PLAN_KEY="[null]" ASSIGNEE="[null]" AUTHOR_LOGIN="[null]" CHECKSUM="ABCDE"
           CREATED_AT="2012-01-05" EFFORT_TO_FIX="3.14" ISSUE_ATTRIBUTES="[null]" ISSUE_CLOSE_DATE="[null]" ISSUE_CREATION_DATE="2012-01-05"
           ISSUE_UPDATE_DATE="2012-01-05" LINE="1234" MANUAL_SEVERITY="[false]" MESSAGE="the message" REPORTER="[null]"
           RESOLUTION="[null]" STATUS="OPEN" UPDATED_AT="2012-01-05"/>
-  <issue_changes ID="1" ISSUE_KEY="ABCDE"/>
-</dataset>
\ No newline at end of file
+  <issue_changes id="1" KEE="ABCDE" ISSUE_KEY="[ignore]" CHANGE_TYPE="comment" CHANGE_DATA="a comment" USER_LOGIN="fabrice" CREATED_AT="2012-04-28" UPDATED_AT="2012-04-29"/>
+</dataset>
diff --git a/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations_result.xml b/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations_result.xml
index 8c0330bcb15..cafbaaf179e 100644
--- a/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations_result.xml
+++ b/sonar-server/src/test/resources/org/sonar/server/db/migrations/violation/ViolationMigrationTest/migrate_violations_result.xml
@@ -1,4 +1,47 @@
 <dataset>
+
+  <snapshots id="10" root_project_id="10" project_id="11"/>
+  <users id="200" login="fabrice"/>
+  <users id="201" login="julien"/>
+  <action_plans id="999" kee="PLAN-999"/>
+
+  <!-- violation without review -->
+  <!--<rule_failures id="1" snapshot_id="10" rule_id="20" failure_level="2" message="the message" line="1234" cost="3.14"-->
+                 <!--created_at="2012-01-05" checksum="ABCDE" permanent_id="1"/>-->
+
+  <!-- violation with review -->
+  <!--<rule_failures id="2" snapshot_id="10" rule_id="22" failure_level="2" message="another message" line="[null]" cost="[null]"-->
+                 <!--created_at="2012-01-05" checksum="FGHIJ" permanent_id="2"/>-->
+  <reviews id="1" rule_failure_permanent_id="2" MANUAL_VIOLATION="[false]" MANUAL_SEVERITY="[true]" SEVERITY="BLOCKER" UPDATED_AT="2013-05-18"
+           STATUS="OPEN" RESOLUTION="[null]" USER_ID="[null]" ASSIGNEE_ID="201"/>
+
+  <review_comments ID="1" REVIEW_ID="1" USER_ID="200" REVIEW_TEXT="a comment" CREATED_AT="2012-04-28" UPDATED_AT="2012-04-29"/>
+
+  <!-- comment by unknown user -->
+  <review_comments ID="2" REVIEW_ID="1" USER_ID="999" REVIEW_TEXT="to be ignored because unknown user" CREATED_AT="2012-04-28" UPDATED_AT="2012-04-29"/>
+
+  <!-- manual violation -->
+  <!--<rule_failures id="3" snapshot_id="10" rule_id="22" failure_level="2" message="another message" line="[null]" cost="[null]"-->
+                 <!--created_at="2012-01-05" checksum="FGHIJ" permanent_id="3"/>-->
+  <reviews id="2" rule_failure_permanent_id="3" MANUAL_VIOLATION="[true]" MANUAL_SEVERITY="[true]" SEVERITY="BLOCKER" UPDATED_AT="2013-05-18"
+           STATUS="RESOLVED" RESOLUTION="FIXED" USER_ID="200" ASSIGNEE_ID="201"/>
+
+  <action_plans_reviews review_id="1" action_plan_id="999"/>
+
+
+
+  <!-- Already there from a past migration that crashed -->
+  <issues ID="1" COMPONENT_ID="11" ROOT_COMPONENT_ID="10" RULE_ID="20" SEVERITY="MINOR" KEE="[ignore]"
+          ACTION_PLAN_KEY="[null]" ASSIGNEE="[null]" AUTHOR_LOGIN="[null]" CHECKSUM="ABCDE"
+          CREATED_AT="2012-01-05" EFFORT_TO_FIX="3.14" ISSUE_ATTRIBUTES="[null]" ISSUE_CLOSE_DATE="[null]" ISSUE_CREATION_DATE="2012-01-05"
+          ISSUE_UPDATE_DATE="2012-01-05" LINE="1234" MANUAL_SEVERITY="[false]" MESSAGE="the message" REPORTER="[null]"
+          RESOLUTION="[null]" STATUS="OPEN" UPDATED_AT="2012-01-05"/>
+  <issue_changes id="1" KEE="ABCDE" ISSUE_KEY="[ignore]" CHANGE_TYPE="comment" CHANGE_DATA="a comment" USER_LOGIN="fabrice" CREATED_AT="2012-04-28" UPDATED_AT="2012-04-29"/>
+
+
+
+
+
   <issues ID="2" COMPONENT_ID="11" ROOT_COMPONENT_ID="10" RULE_ID="20" SEVERITY="MINOR" KEE="[ignore]"
           ACTION_PLAN_KEY="[null]" ASSIGNEE="[null]" AUTHOR_LOGIN="[null]" CHECKSUM="ABCDE"
           CREATED_AT="2012-01-05" EFFORT_TO_FIX="3.14" ISSUE_ATTRIBUTES="[null]" ISSUE_CLOSE_DATE="[null]" ISSUE_CREATION_DATE="2012-01-05"