diff options
| author | Simon Brandhof <simon.brandhof@gmail.com> | 2013-07-04 19:23:31 +0200 |
|---|---|---|
| committer | Simon Brandhof <simon.brandhof@gmail.com> | 2013-07-04 19:26:01 +0200 |
| commit | ccb95087fd7aab08d2e8e057eecd21797ed7f9b6 (patch) | |
| tree | 2bcb2aaccbe363ed93dea95e0d96e182692c264c /sonar-core | |
| parent | a60604f5d250121fd594bb3d331a0f85fe889797 (diff) | |
| download | sonarqube-ccb95087fd7aab08d2e8e057eecd21797ed7f9b6.tar.gz sonarqube-ccb95087fd7aab08d2e8e057eecd21797ed7f9b6.zip | |
SONAR-4470 Performance issue when migrating DB from 3.5 to 3.6 (mainly with Oracle)
Diffstat (limited to 'sonar-core')
17 files changed, 414 insertions, 373 deletions
diff --git a/sonar-core/pom.xml b/sonar-core/pom.xml index c7c65fa9e66..7047fc7d230 100644 --- a/sonar-core/pom.xml +++ b/sonar-core/pom.xml @@ -61,6 +61,10 @@ <artifactId>commons-dbcp</artifactId> </dependency> <dependency> + <groupId>commons-dbutils</groupId> + <artifactId>commons-dbutils</artifactId> + </dependency> + <dependency> <groupId>org.codehaus.plexus</groupId> <artifactId>plexus-classworlds</artifactId> </dependency> diff --git a/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterExecutor.java b/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterExecutor.java index 2e90c431243..81168944cf2 100644 --- a/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterExecutor.java +++ b/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterExecutor.java @@ -20,10 +20,10 @@ package org.sonar.core.measure; import com.google.common.base.Strings; +import org.apache.commons.dbutils.DbUtils; import org.apache.ibatis.session.SqlSession; import org.sonar.api.ServerComponent; import org.sonar.core.persistence.Database; -import org.sonar.core.persistence.DatabaseUtils; import org.sonar.core.persistence.MyBatis; import org.sonar.core.resource.ResourceDao; @@ -67,7 +67,7 @@ public class MeasureFilterExecutor implements ServerComponent { } finally { MyBatis.closeQuietly(session); // connection is supposed to be closed by the session - DatabaseUtils.closeQuietly(connection); + DbUtils.closeQuietly(connection); } return rows; diff --git a/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterSql.java b/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterSql.java index 8545d98be8f..bed09124a09 100644 --- a/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterSql.java +++ b/sonar-core/src/main/java/org/sonar/core/measure/MeasureFilterSql.java @@ -22,10 +22,10 @@ package org.sonar.core.measure; import com.google.common.base.Function; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; +import org.apache.commons.dbutils.DbUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.sonar.core.persistence.Database; -import org.sonar.core.persistence.DatabaseUtils; import org.sonar.core.resource.SnapshotDto; import javax.annotation.Nullable; @@ -67,8 +67,8 @@ class MeasureFilterSql { return process(rs); } finally { - DatabaseUtils.closeQuietly(rs); - DatabaseUtils.closeQuietly(statement); + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(statement); } } diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseMigrator.java b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseMigrator.java deleted file mode 100644 index b15db5f3711..00000000000 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseMigrator.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2013 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.core.persistence; - -import org.apache.ibatis.session.SqlSession; -import org.slf4j.LoggerFactory; -import org.sonar.api.ServerComponent; - -import java.sql.Connection; - -/** - * Restore schema by executing DDL scripts. Only H2 database is supported. - * Other databases are created by Ruby on Rails migrations. - * - * @since 2.12 - */ -public class DatabaseMigrator implements ServerComponent { - - private MyBatis myBatis; - private Database database; - - public DatabaseMigrator(MyBatis myBatis, Database database) { - this.myBatis = myBatis; - this.database = database; - } - - /** - * @return true if the database has been created, false if this database is not supported - */ - public boolean createDatabase() { - if (!DdlUtils.supportsDialect(database.getDialect().getId())) { - return false; - } - - LoggerFactory.getLogger(getClass()).info("Create database"); - SqlSession session = null; - Connection connection = null; - try { - session = myBatis.openSession(); - connection = session.getConnection(); - DdlUtils.createSchema(connection, database.getDialect().getId()); - } finally { - MyBatis.closeQuietly(session); - - // The connection is probably already closed by session.close() - // but it's not documented in mybatis javadoc. - DatabaseUtils.closeQuietly(connection); - } - return true; - } -} diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseUtils.java b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseUtils.java deleted file mode 100644 index aba584bcaa3..00000000000 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseUtils.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2013 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.core.persistence; - -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -/** - * @since 2.13 - */ -public final class DatabaseUtils { - private DatabaseUtils() { - } - - /** - * List of all the tables. - * This list is hardcoded because we didn't succeed in using java.sql.DatabaseMetaData#getTables() in the same way - * for all the supported databases, particularly due to Oracle results. - */ - static final String[] TABLE_NAMES = { - "action_plans", - "active_dashboards", - "active_rules", - "active_rule_changes", - "active_rule_parameters", - "active_rule_param_changes", - "alerts", - "authors", - "characteristics", - "characteristic_edges", - "characteristic_properties", - "dashboards", - "dependencies", - "duplications_index", - "events", - "graphs", - "groups", - "groups_users", - "group_roles", - "issues", - "issue_changes", - "issue_filters", - "issue_filter_favourites", - "loaded_templates", - "manual_measures", - "measure_data", - "measure_filters", - "measure_filter_favourites", - "metrics", - "notifications", - "permission_templates", - "perm_templates_users", - "perm_templates_groups", - "projects", - "project_links", - "project_measures", - "properties", - "quality_models", - "resource_index", - "rules", - "rules_parameters", - "rules_profiles", - "semaphores", - "schema_migrations", - "snapshots", - "snapshot_sources", - "snapshot_data", - "users", - "user_roles", - "widgets", - "widget_properties"}; - - public static void closeQuietly(@Nullable Connection connection) { - if (connection != null) { - try { - connection.close(); - } catch (SQLException e) { - LoggerFactory.getLogger(DatabaseUtils.class).warn("Fail to close connection", e); - // ignore - } - } - } - - public static void closeQuietly(@Nullable Statement stmt) { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - LoggerFactory.getLogger(DatabaseUtils.class).warn("Fail to close statement", e); - // ignore - } - } - } - - public static void closeQuietly(@Nullable ResultSet rs) { - if (rs != null) { - try { - rs.close(); - } catch (SQLException e) { - LoggerFactory.getLogger(DatabaseUtils.class).warn("Fail to close result set", e); - // ignore - } - } - } -} diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java index 16bde3cd397..e77f001bd6b 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java @@ -20,6 +20,7 @@ package org.sonar.core.persistence; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import org.apache.ibatis.session.SqlSession; import org.sonar.api.BatchComponent; import org.sonar.api.ServerComponent; @@ -38,6 +39,65 @@ public class DatabaseVersion implements BatchComponent, ServerComponent { UP_TO_DATE, REQUIRES_UPGRADE, REQUIRES_DOWNGRADE, FRESH_INSTALL } + /** + * List of all the tables. + * This list is hardcoded because we didn't succeed in using java.sql.DatabaseMetaData#getTables() in the same way + * for all the supported databases, particularly due to Oracle results. + */ + public static final List<String> TABLES = ImmutableList.of( + "action_plans", + "active_dashboards", + "active_rules", + "active_rule_changes", + "active_rule_parameters", + "active_rule_param_changes", + "alerts", + "authors", + "characteristics", + "characteristic_edges", + "characteristic_properties", + "dashboards", + "dependencies", + "duplications_index", + "events", + "graphs", + "groups", + "groups_users", + "group_roles", + "issues", + "issue_changes", + "issue_filters", + "issue_filter_favourites", + "loaded_templates", + "manual_measures", + "measure_data", + "measure_filters", + "measure_filter_favourites", + "metrics", + "notifications", + "permission_templates", + "perm_templates_users", + "perm_templates_groups", + "projects", + "project_links", + "project_measures", + "properties", + "quality_models", + "resource_index", + "rules", + "rules_parameters", + "rules_profiles", + "semaphores", + "schema_migrations", + "snapshots", + "snapshot_sources", + "snapshot_data", + "users", + "user_roles", + "widgets", + "widget_properties" + ); + private MyBatis mybatis; public DatabaseVersion(MyBatis mybatis) { diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java b/sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java index 52e2dbd1c41..2a1fb7d9d1b 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.Lists; import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbutils.DbUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,12 +90,12 @@ public class DbTemplate implements ServerComponent { LOG.error("Fail to copy table " + table, e); throw new IllegalStateException("Fail to copy table " + table, e); } finally { - DatabaseUtils.closeQuietly(destStatement); - DatabaseUtils.closeQuietly(destResultSet); - DatabaseUtils.closeQuietly(destConnection); - DatabaseUtils.closeQuietly(sourceResultSet); - DatabaseUtils.closeQuietly(sourceStatement); - DatabaseUtils.closeQuietly(sourceConnection); + DbUtils.closeQuietly(destStatement); + DbUtils.closeQuietly(destResultSet); + DbUtils.closeQuietly(destConnection); + DbUtils.closeQuietly(sourceResultSet); + DbUtils.closeQuietly(sourceStatement); + DbUtils.closeQuietly(sourceConnection); } return this; @@ -158,9 +159,9 @@ public class DbTemplate implements ServerComponent { LOG.error("Fail to get row count for table " + table, e); throw new SonarException("Fail to get row count for table " + table, e); } finally { - DatabaseUtils.closeQuietly(resultSet); - DatabaseUtils.closeQuietly(statement); - DatabaseUtils.closeQuietly(connection); + DbUtils.closeQuietly(resultSet); + DbUtils.closeQuietly(statement); + DbUtils.closeQuietly(connection); } } @@ -175,8 +176,8 @@ public class DbTemplate implements ServerComponent { LOG.error("Fail to truncate table " + table, e); throw new SonarException("Fail to truncate table " + table, e); } finally { - DatabaseUtils.closeQuietly(statement); - DatabaseUtils.closeQuietly(connection); + DbUtils.closeQuietly(statement); + DbUtils.closeQuietly(connection); } return this; @@ -200,7 +201,7 @@ public class DbTemplate implements ServerComponent { LOG.error("Fail to createSchema local database schema", e); throw new SonarException("Fail to createSchema local database schema", e); } finally { - DatabaseUtils.closeQuietly(connection); + DbUtils.closeQuietly(connection); } return this; diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DdlUtils.java b/sonar-core/src/main/java/org/sonar/core/persistence/DdlUtils.java index 94ac7774d43..0ea501c2083 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DdlUtils.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DdlUtils.java @@ -48,7 +48,7 @@ public final class DdlUtils { executeScript(connection, "org/sonar/core/persistence/rows-" + dialect + ".sql"); } - private static void executeScript(Connection connection, String path) { + public static void executeScript(Connection connection, String path) { ScriptRunner scriptRunner = newScriptRunner(connection); try { scriptRunner.runScript(Resources.getResourceAsReader(path)); diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DefaultDatabase.java b/sonar-core/src/main/java/org/sonar/core/persistence/DefaultDatabase.java index 3b8d5c5c2d6..5d0752f0d4d 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DefaultDatabase.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DefaultDatabase.java @@ -22,6 +22,7 @@ package org.sonar.core.persistence; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbcp.BasicDataSourceFactory; +import org.apache.commons.dbutils.DbUtils; import org.apache.commons.lang.StringUtils; import org.hibernate.cfg.Environment; import org.slf4j.Logger; @@ -141,7 +142,7 @@ public class DefaultDatabase implements Database { } catch (Exception e) { LOG.error("Can not connect to database. Please check connectivity and settings (see the properties prefixed by 'sonar.jdbc.').", e); } finally { - DatabaseUtils.closeQuietly(connection); + DbUtils.closeQuietly(connection); } } diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/AbstractDaoTestCase.java b/sonar-core/src/test/java/org/sonar/core/persistence/AbstractDaoTestCase.java index a6e1e1b5d05..4d208829db1 100644 --- a/sonar-core/src/test/java/org/sonar/core/persistence/AbstractDaoTestCase.java +++ b/sonar-core/src/test/java/org/sonar/core/persistence/AbstractDaoTestCase.java @@ -60,7 +60,7 @@ import java.util.Properties; import static org.junit.Assert.fail; public abstract class AbstractDaoTestCase { - private static Logger LOG = LoggerFactory.getLogger(AbstractDaoTestCase.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractDaoTestCase.class); private static Database database; private static DatabaseCommands databaseCommands; private static IDatabaseTester databaseTester; @@ -80,7 +80,7 @@ public abstract class AbstractDaoTestCase { if (hasDialect) { database = new DefaultDatabase(settings); } else { - database = new H2Database("h2Tests"); + database = new H2Database("h2Tests", true); } database.start(); LOG.info("Test Database: " + database); diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/DatabaseCommands.java b/sonar-core/src/test/java/org/sonar/core/persistence/DatabaseCommands.java index 2205ac573f5..21fb4c923c1 100644 --- a/sonar-core/src/test/java/org/sonar/core/persistence/DatabaseCommands.java +++ b/sonar-core/src/test/java/org/sonar/core/persistence/DatabaseCommands.java @@ -119,9 +119,14 @@ public abstract class DatabaseCommands { connection.setAutoCommit(false); Statement statement = connection.createStatement(); - for (String table : DatabaseUtils.TABLE_NAMES) { - statement.executeUpdate("TRUNCATE TABLE " + table); - connection.commit(); + for (String table : DatabaseVersion.TABLES) { + try { + statement.executeUpdate("TRUNCATE TABLE " + table); + connection.commit(); + } catch (Exception e) { + // ignore + connection.rollback(); + } } statement.close(); @@ -133,7 +138,7 @@ public abstract class DatabaseCommands { connection.setAutoCommit(false); Statement statement = connection.createStatement(); - for (String table : DatabaseUtils.TABLE_NAMES) { + for (String table : DatabaseVersion.TABLES) { try { ResultSet result = statement.executeQuery("SELECT CASE WHEN MAX(ID) IS NULL THEN 1 ELSE MAX(ID)+1 END FROM " + table); result.next(); diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/DatabaseUtilsTest.java b/sonar-core/src/test/java/org/sonar/core/persistence/DatabaseUtilsTest.java deleted file mode 100644 index 28e1ef948f7..00000000000 --- a/sonar-core/src/test/java/org/sonar/core/persistence/DatabaseUtilsTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2013 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.core.persistence; - -import org.junit.Test; -import org.sonar.core.persistence.dialect.Oracle; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -import static org.fest.assertions.Assertions.assertThat; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -public class DatabaseUtilsTest extends AbstractDaoTestCase { - - @Test - public void should_close_connection() throws SQLException { - Connection connection = getConnection(); - assertThat(isClosed(connection)).isFalse(); - - DatabaseUtils.closeQuietly(connection); - assertThat(isClosed(connection)).isTrue(); - } - - @Test - public void should_support_null_connection() { - DatabaseUtils.closeQuietly((Connection) null); - // no failure - } - - @Test - public void should_close_statement_and_resultset() throws SQLException { - Connection connection = getConnection(); - try { - PreparedStatement statement = connection.prepareStatement(selectDual()); - ResultSet rs = statement.executeQuery(); - - DatabaseUtils.closeQuietly(rs); - DatabaseUtils.closeQuietly(statement); - - assertThat(isClosed(statement)).isTrue(); - assertThat(isClosed(rs)).isTrue(); - } finally { - DatabaseUtils.closeQuietly(connection); - } - } - - @Test - public void should_not_fail_on_connection_errors() throws SQLException { - Connection connection = mock(Connection.class); - doThrow(new SQLException()).when(connection).close(); - - DatabaseUtils.closeQuietly(connection); - - // no failure - verify(connection).close(); // just to be sure - } - - @Test - public void should_not_fail_on_statement_errors() throws SQLException { - Statement statement = mock(Statement.class); - doThrow(new SQLException()).when(statement).close(); - - DatabaseUtils.closeQuietly(statement); - - // no failure - verify(statement).close(); // just to be sure - } - - @Test - public void should_not_fail_on_resulset_errors() throws SQLException { - ResultSet rs = mock(ResultSet.class); - doThrow(new SQLException()).when(rs).close(); - - DatabaseUtils.closeQuietly(rs); - - // no failure - verify(rs).close(); // just to be sure - } - - /** - * Connection.isClosed() has been introduced in java 1.6 - */ - private boolean isClosed(Connection c) { - try { - c.createStatement().execute(selectDual()); - return false; - } catch (Exception e) { - return true; - } - } - - /** - * Statement.isClosed() has been introduced in java 1.6 - */ - private boolean isClosed(Statement s) { - try { - s.execute("SELECT 1"); - return false; - } catch (Exception e) { - return true; - } - } - - /** - * ResultSet.isClosed() has been introduced in java 1.6 - */ - private boolean isClosed(ResultSet rs) { - try { - rs.next(); - return false; - } catch (Exception e) { - return true; - } - } - - private String selectDual() { - String sql = "SELECT 1"; - if (Oracle.ID.equals(getDatabase().getDialect().getId())) { - sql = "SELECT 1 FROM DUAL"; - } - return sql; - } -} diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/H2Database.java b/sonar-core/src/test/java/org/sonar/core/persistence/H2Database.java index 4536993eb1d..31c6619aea7 100644 --- a/sonar-core/src/test/java/org/sonar/core/persistence/H2Database.java +++ b/sonar-core/src/test/java/org/sonar/core/persistence/H2Database.java @@ -20,6 +20,7 @@ package org.sonar.core.persistence; import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbutils.DbUtils; import org.hibernate.cfg.Environment; import org.sonar.core.persistence.dialect.Dialect; import org.sonar.core.persistence.dialect.H2; @@ -38,18 +39,22 @@ import java.util.Properties; */ public class H2Database implements Database { private final String name; + private final boolean createSchema; private BasicDataSource datasource; /** * IMPORTANT: change DB name in order to not conflict with {@link DefaultDatabaseTest} */ - public H2Database(String name) { + public H2Database(String name, boolean createSchema) { this.name = name; + this.createSchema = createSchema; } public H2Database start() { startDatabase(); - createSchema(); + if (createSchema) { + createSchema(); + } return this; } @@ -70,10 +75,24 @@ public class H2Database implements Database { try { connection = datasource.getConnection(); DdlUtils.createSchema(connection, "h2"); + } catch (SQLException e) { throw new IllegalStateException("Fail to create schema", e); } finally { - DatabaseUtils.closeQuietly(connection); + DbUtils.closeQuietly(connection); + } + } + + public void executeScript(String classloaderPath) { + Connection connection = null; + try { + connection = datasource.getConnection(); + DdlUtils.executeScript(connection, classloaderPath); + + } catch (SQLException e) { + throw new IllegalStateException("Fail to execute script: " + classloaderPath, e); + } finally { + DbUtils.closeQuietly(connection); } } diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/H2DatabaseTest.java b/sonar-core/src/test/java/org/sonar/core/persistence/H2DatabaseTest.java index 67703eda4f5..19ab908c98a 100644 --- a/sonar-core/src/test/java/org/sonar/core/persistence/H2DatabaseTest.java +++ b/sonar-core/src/test/java/org/sonar/core/persistence/H2DatabaseTest.java @@ -30,7 +30,7 @@ import java.sql.SQLException; import static org.fest.assertions.Assertions.assertThat; public class H2DatabaseTest { - H2Database db = new H2Database("sonar2"); + H2Database db = new H2Database("sonar2", true); @Before public void startDb() { diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/MyBatisTest.java b/sonar-core/src/test/java/org/sonar/core/persistence/MyBatisTest.java index d0455791383..3a3bfcad71a 100644 --- a/sonar-core/src/test/java/org/sonar/core/persistence/MyBatisTest.java +++ b/sonar-core/src/test/java/org/sonar/core/persistence/MyBatisTest.java @@ -41,7 +41,7 @@ public class MyBatisTest { @BeforeClass public static void start() { - database = new H2Database("sonar2"); + database = new H2Database("sonar2", true); database.start(); } diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/TestDatabase.java b/sonar-core/src/test/java/org/sonar/core/persistence/TestDatabase.java new file mode 100644 index 00000000000..39d0bf63762 --- /dev/null +++ b/sonar-core/src/test/java/org/sonar/core/persistence/TestDatabase.java @@ -0,0 +1,293 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.persistence; + +import com.google.common.collect.Maps; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.dbutils.DbUtils; +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.text.StrSubstitutor; +import org.dbunit.Assertion; +import org.dbunit.DataSourceDatabaseTester; +import org.dbunit.DatabaseUnitException; +import org.dbunit.IDatabaseTester; +import org.dbunit.assertion.DiffCollectingFailureHandler; +import org.dbunit.assertion.Difference; +import org.dbunit.database.DatabaseConfig; +import org.dbunit.database.IDatabaseConnection; +import org.dbunit.dataset.CompositeDataSet; +import org.dbunit.dataset.IDataSet; +import org.dbunit.dataset.ReplacementDataSet; +import org.dbunit.dataset.xml.FlatXmlDataSet; +import org.dbunit.ext.mssql.InsertIdentityOperation; +import org.dbunit.operation.DatabaseOperation; +import org.junit.rules.ExternalResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.config.Settings; +import org.sonar.core.config.Logback; +import org.sonar.core.persistence.dialect.Dialect; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.fail; + +public class TestDatabase extends ExternalResource { + + private static final Logger LOG = LoggerFactory.getLogger(TestDatabase.class); + + private Database db; + private DatabaseCommands commands; + private IDatabaseTester tester; + private MyBatis myBatis; + private String schemaPath = null; + + + public TestDatabase schema(Class baseClass, String filename) { + String path = StringUtils.replaceChars(baseClass.getCanonicalName(), '.', '/'); + schemaPath = path + "/" + filename; + return this; + } + + @Override + protected void before() throws Throwable { + if (db == null) { + Settings settings = new Settings().setProperties(Maps.fromProperties(System.getProperties())); + if (settings.hasKey("orchestrator.configUrl")) { + loadOrchestratorSettings(settings); + } + for (String key : settings.getKeysStartingWith("sonar.jdbc")) { + LOG.info(key + ": " + settings.getString(key)); + } + boolean hasDialect = settings.hasKey("sonar.jdbc.dialect"); + if (hasDialect) { + db = new DefaultDatabase(settings); + } else { + db = new H2Database("h2Tests" + DigestUtils.md5Hex(StringUtils.defaultString(schemaPath)), schemaPath == null); + } + db.start(); + if (schemaPath != null) { + // will fail if not H2 + ((H2Database) db).executeScript(schemaPath); + } + LOG.info("Test Database: " + db); + + commands = DatabaseCommands.forDialect(db.getDialect()); + tester = new DataSourceDatabaseTester(db.getDataSource()); + + myBatis = new MyBatis(db, settings, new Logback()); + myBatis.start(); + } + commands.truncateDatabase(db.getDataSource()); + } + + + @Override + protected void after() { + if (db != null) { + db.stop(); + db = null; + commands = null; + myBatis = null; + } + } + + public Database database() { + return db; + } + + public Dialect dialect() { + return db.getDialect(); + } + + public DatabaseCommands commands() { + return commands; + } + + public MyBatis myBatis() { + return myBatis; + } + + public Connection openConnection() throws SQLException { + return db.getDataSource().getConnection(); + } + + public void executeUpdateSql(String sql) { + Connection connection = null; + try { + connection = openConnection(); + new QueryRunner().update(connection, sql); + } catch (Exception e) { + throw new IllegalStateException("Fail to execute sql: " + sql); + } finally { + DbUtils.commitAndCloseQuietly(connection); + } + } + + public void prepareDbUnit(Class testClass, String... testNames) { + InputStream[] streams = new InputStream[testNames.length]; + try { + for (int i = 0; i < testNames.length; i++) { + String path = "/" + testClass.getName().replace('.', '/') + "/" + testNames[i]; + streams[i] = testClass.getResourceAsStream(path); + if (streams[i] == null) { + throw new IllegalStateException("DbUnit file not found: " + path); + } + } + + prepareDbUnit(streams); + commands.resetPrimaryKeys(db.getDataSource()); + } catch (SQLException e) { + throw translateException("Could not setup DBUnit data", e); + } finally { + for (InputStream stream : streams) { + IOUtils.closeQuietly(stream); + } + } + } + + private void prepareDbUnit(InputStream... dataSetStream) { + IDatabaseConnection connection = null; + try { + IDataSet[] dataSets = new IDataSet[dataSetStream.length]; + for (int i = 0; i < dataSetStream.length; i++) { + dataSets[i] = dbUnitDataSet(dataSetStream[i]); + } + tester.setDataSet(new CompositeDataSet(dataSets)); + connection = dbUnitConnection(); + new InsertIdentityOperation(DatabaseOperation.INSERT).execute(connection, tester.getDataSet()); + } catch (Exception e) { + throw translateException("Could not setup DBUnit data", e); + } finally { + closeQuietly(connection); + } + } + + public void assertDbUnit(Class testClass, String filename, String... tables) { + IDatabaseConnection connection = null; + try { + connection = dbUnitConnection(); + IDataSet dataSet = connection.createDataSet(); + String path = "/" + testClass.getName().replace('.', '/') + "/" + filename; + IDataSet expectedDataSet = dbUnitDataSet(testClass.getResourceAsStream(path)); + for (String table : tables) { + DiffCollectingFailureHandler diffHandler = new DiffCollectingFailureHandler(); + + Assertion.assertEquals(expectedDataSet.getTable(table), dataSet.getTable(table), diffHandler); + // Evaluate the differences and ignore some column values + List diffList = diffHandler.getDiffList(); + for (Object o : diffList) { + Difference diff = (Difference) o; + if (!"[ignore]".equals(diff.getExpectedValue())) { + throw new DatabaseUnitException(diff.toString()); + } + } + } + } catch (DatabaseUnitException e) { + fail(e.getMessage()); + } catch (Exception e) { + throw translateException("Error while checking results", e); + } finally { + closeQuietly(connection); + } + } + + private IDataSet dbUnitDataSet(InputStream stream) { + try { + ReplacementDataSet dataSet = new ReplacementDataSet(new FlatXmlDataSet(stream)); + dataSet.addReplacementObject("[null]", null); + dataSet.addReplacementObject("[false]", Boolean.FALSE); + dataSet.addReplacementObject("[true]", Boolean.TRUE); + return dataSet; + } catch (Exception e) { + throw translateException("Could not read the dataset stream", e); + } + } + + private IDatabaseConnection dbUnitConnection() { + try { + IDatabaseConnection connection = tester.getConnection(); + connection.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, commands.getDbUnitFactory()); + return connection; + } catch (Exception e) { + throw translateException("Error while getting connection", e); + } + } + + private void closeQuietly(IDatabaseConnection connection) { + try { + if (connection != null) { + connection.close(); + } + } catch (SQLException e) { + // ignore + } + } + + private static RuntimeException translateException(String msg, Exception cause) { + RuntimeException runtimeException = new RuntimeException(String.format("%s: [%s] %s", msg, cause.getClass().getName(), cause.getMessage())); + runtimeException.setStackTrace(cause.getStackTrace()); + return runtimeException; + } + + private void loadOrchestratorSettings(Settings settings) throws URISyntaxException, IOException { + String url = settings.getString("orchestrator.configUrl"); + URI uri = new URI(url); + InputStream input = null; + try { + if (url.startsWith("file:")) { + File file = new File(uri); + input = FileUtils.openInputStream(file); + } else { + HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(); + int responseCode = connection.getResponseCode(); + if (responseCode >= 400) { + throw new IllegalStateException("Fail to request: " + uri + ". Status code=" + responseCode); + } + + input = connection.getInputStream(); + + } + Properties props = new Properties(); + props.load(input); + settings.addProperties(props); + for (Map.Entry<String, String> entry : settings.getProperties().entrySet()) { + String interpolatedValue = StrSubstitutor.replace(entry.getValue(), System.getenv(), "${", "}"); + settings.setProperty(entry.getKey(), interpolatedValue); + } + } finally { + IOUtils.closeQuietly(input); + } + } + +} diff --git a/sonar-core/src/test/java/org/sonar/jpa/test/AbstractDbUnitTestCase.java b/sonar-core/src/test/java/org/sonar/jpa/test/AbstractDbUnitTestCase.java index 097ff170350..7998f074bd7 100644 --- a/sonar-core/src/test/java/org/sonar/jpa/test/AbstractDbUnitTestCase.java +++ b/sonar-core/src/test/java/org/sonar/jpa/test/AbstractDbUnitTestCase.java @@ -63,7 +63,7 @@ public abstract class AbstractDbUnitTestCase { @Before public void startDatabase() throws SQLException { if (database == null) { - database = new H2Database("sonarHibernate"); + database = new H2Database("sonarHibernate", true); database.start(); databaseCommands = DatabaseCommands.forDialect(database.getDialect()); |