From 525b465edfbfe72f0e0959b75112e08d9420983e Mon Sep 17 00:00:00 2001 From: David Gageot Date: Tue, 23 Oct 2012 20:36:54 +0200 Subject: [PATCH] SONAR-3895 Better Local mode --- sonar-batch/pom.xml | 5 -- .../org/sonar/batch/local/LocalDatabase.java | 76 +++++----------- .../sonar/core/persistence/DbTemplate.java | 85 +++++++++--------- .../persistence/LocalDatabaseFactory.java | 90 +++++++++++++++++++ .../persistence/LocalDatabaseFactoryTest.java | 75 ++++++++++++++++ .../should_create_database.xml | 6 ++ .../org/sonar/server/platform/Platform.java | 2 +- .../app/controllers/api/synchro_controller.rb | 10 +-- 8 files changed, 238 insertions(+), 111 deletions(-) rename sonar-server/src/main/java/org/sonar/server/database/LocalDatabaseFactory.java => sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java (66%) create mode 100644 sonar-core/src/main/java/org/sonar/core/persistence/LocalDatabaseFactory.java create mode 100644 sonar-core/src/test/java/org/sonar/core/persistence/LocalDatabaseFactoryTest.java create mode 100644 sonar-core/src/test/resources/org/sonar/core/persistence/LocalDatabaseFactoryTest/should_create_database.xml diff --git a/sonar-batch/pom.xml b/sonar-batch/pom.xml index 6a1a222b901..7255d9719b2 100644 --- a/sonar-batch/pom.xml +++ b/sonar-batch/pom.xml @@ -56,11 +56,6 @@ commons-lang commons-lang - - com.google.code.gson - gson - 2.2.2 - diff --git a/sonar-batch/src/main/java/org/sonar/batch/local/LocalDatabase.java b/sonar-batch/src/main/java/org/sonar/batch/local/LocalDatabase.java index a8e1877e6f6..51d75d53730 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/local/LocalDatabase.java +++ b/sonar-batch/src/main/java/org/sonar/batch/local/LocalDatabase.java @@ -19,11 +19,8 @@ */ package org.sonar.batch.local; -import com.google.common.io.Closeables; -import com.google.gson.Gson; -import org.apache.commons.dbcp.BasicDataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.google.common.io.Files; +import com.google.common.io.InputSupplier; import org.sonar.api.BatchComponent; import org.sonar.api.config.Settings; import org.sonar.api.database.DatabaseProperties; @@ -31,19 +28,18 @@ import org.sonar.api.platform.Server; import org.sonar.api.utils.HttpDownloader; import org.sonar.api.utils.SonarException; import org.sonar.batch.bootstrap.LocalMode; +import org.sonar.batch.bootstrap.TempDirectories; +import java.io.File; +import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URI; -import java.sql.SQLException; /** * @since 3.4 */ public class LocalDatabase implements BatchComponent { - private static final Logger LOG = LoggerFactory.getLogger(LocalDatabase.class); - - public static final String API_SYNCHRO = "/api/synchro"; + private static final String API_SYNCHRO = "/api/synchro"; private static final String DIALECT = "h2"; private static final String DRIVER = "org.h2.Driver"; private static final String URL = "jdbc:h2:"; @@ -54,13 +50,14 @@ public class LocalDatabase implements BatchComponent { private final Settings settings; private final Server server; private final HttpDownloader httpDownloader; - private BasicDataSource dataSource; + private final TempDirectories tempDirectories; - public LocalDatabase(LocalMode localMode, Settings settings, Server server, HttpDownloader httpDownloader) { + public LocalDatabase(LocalMode localMode, Settings settings, Server server, HttpDownloader httpDownloader, TempDirectories tempDirectories) { this.localMode = localMode; this.settings = settings; this.server = server; this.httpDownloader = httpDownloader; + this.tempDirectories = tempDirectories; } public void start() { @@ -68,58 +65,31 @@ public class LocalDatabase implements BatchComponent { return; } - LOG.info("Download database"); - Path path = downloadDatabase(); + File file = tempDirectories.getFile("local", "db.h2.db"); + String h2DatabasePath = file.getAbsolutePath().replaceAll(".h2.db", ""); - LOG.info("Starting local database"); - replaceSettings(path); - configureDataSource(path); + downloadDatabase(file); + replaceSettings(h2DatabasePath); } - private Path downloadDatabase() { - InputStream stream = null; + private void downloadDatabase(File toFile) { try { - stream = httpDownloader.openStream(URI.create(server.getURL() + API_SYNCHRO)); - return new Gson().fromJson(new InputStreamReader(stream), Path.class); - } finally { - Closeables.closeQuietly(stream); + Files.copy(new InputSupplier() { + public InputStream getInput() { + return httpDownloader.openStream(URI.create(server.getURL() + API_SYNCHRO)); + } + }, toFile); + } catch (IOException e) { + throw new SonarException("Unable to download database", e); } } - static class Path { - String path; - - String getName() { - return path.replaceAll(".h2.db", ""); - } - } - - public void stop() { - try { - dataSource.close(); - } catch (SQLException e) { - // Ignore error - } - } - - private void replaceSettings(Path path) { + private void replaceSettings(String h2DatabasePath) { settings .setProperty(DatabaseProperties.PROP_DIALECT, DIALECT) .setProperty(DatabaseProperties.PROP_DRIVER, DRIVER) .setProperty(DatabaseProperties.PROP_USER, USER) .setProperty(DatabaseProperties.PROP_PASSWORD, PASSWORD) - .setProperty(DatabaseProperties.PROP_URL, URL + path.getName()); - } - - private void configureDataSource(Path path) { - try { - dataSource = new BasicDataSource(); - dataSource.setDriverClassName(DRIVER); - dataSource.setUsername(USER); - dataSource.setPassword(PASSWORD); - dataSource.setUrl(URL + path.getName()); - } catch (Exception e) { - throw new SonarException("Fail to start local database", e); - } + .setProperty(DatabaseProperties.PROP_URL, URL + h2DatabasePath); } } diff --git a/sonar-server/src/main/java/org/sonar/server/database/LocalDatabaseFactory.java b/sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java similarity index 66% rename from sonar-server/src/main/java/org/sonar/server/database/LocalDatabaseFactory.java rename to sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java index 7a08c7e17ac..ec40ffee5eb 100644 --- a/sonar-server/src/main/java/org/sonar/server/database/LocalDatabaseFactory.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DbTemplate.java @@ -17,7 +17,7 @@ * License along with Sonar; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 */ -package org.sonar.server.database; +package org.sonar.core.persistence; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.lang.StringUtils; @@ -25,55 +25,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.ServerComponent; import org.sonar.api.utils.SonarException; -import org.sonar.core.persistence.Database; -import org.sonar.core.persistence.DdlUtils; import javax.sql.DataSource; -import java.io.File; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -public class LocalDatabaseFactory implements ServerComponent { - private static final Logger LOG = LoggerFactory.getLogger(LocalDatabaseFactory.class); +public class DbTemplate implements ServerComponent { + private static final Logger LOG = LoggerFactory.getLogger(DbTemplate.class); - private static final String H2_DIALECT = "h2"; - private static final String H2_DRIVER = "org.h2.Driver"; - private static final String H2_URL = "jdbc:h2:"; - private static final String H2_USER = "sonar"; - private static final String H2_PASSWORD = "sonar"; - - private Database database; - - public LocalDatabaseFactory(Database database) { - this.database = database; - } - - public String createDatabaseForLocalMode() throws SQLException { - String name = "/tmp/" + System.nanoTime(); - - DataSource source = database.getDataSource(); - BasicDataSource destination = dataSource(H2_DRIVER, H2_USER, H2_PASSWORD, H2_URL + name); - - create(destination, H2_DIALECT); - - copyTable(source, destination, "PROPERTIES", "SELECT * FROM PROPERTIES WHERE (USER_ID IS NULL) AND (RESOURCE_ID IS NULL)"); - copyTable(source, destination, "RULES_PROFILES", "SELECT * FROM RULES_PROFILES"); - copyTable(source, destination, "RULES", "SELECT * FROM RULES"); - copyTable(source, destination, "RULES_PARAMETERS", "SELECT * FROM RULES_PARAMETERS"); - copyTable(source, destination, "ACTIVE_RULES", "SELECT * FROM ACTIVE_RULES"); - copyTable(source, destination, "ACTIVE_RULE_PARAMETERS", "SELECT * FROM ACTIVE_RULE_PARAMETERS"); - copyTable(source, destination, "METRICS", "SELECT * FROM METRICS"); - - destination.close(); - - return new File(name + ".h2.db").getAbsolutePath(); - } - - private void copyTable(DataSource source, DataSource dest, String table, String query) throws SQLException { + public DbTemplate copyTable(DataSource source, DataSource dest, String table, String query) { LOG.info("Copy table " + table); int colCount = getColumnCount(source, table); @@ -105,6 +69,8 @@ public class LocalDatabaseFactory implements ServerComponent { destStatement.executeBatch(); destConnection.commit(); destStatement.close(); + } catch (SQLException e) { + throw new SonarException("Fail to copy table " + table, e); } finally { closeQuietly(destResultSet); closeQuietly(destConnection); @@ -112,9 +78,11 @@ public class LocalDatabaseFactory implements ServerComponent { closeQuietly(sourceStatement); closeQuietly(sourceConnection); } + + return this; } - private int getColumnCount(DataSource dataSource, String table) throws SQLException { + public int getColumnCount(DataSource dataSource, String table) { Connection connection = null; ResultSet metaData = null; try { @@ -127,26 +95,51 @@ public class LocalDatabaseFactory implements ServerComponent { } return nbColumns; + } catch (SQLException e) { + throw new SonarException("Fail to get column count for table " + table, e); } finally { closeQuietly(metaData); closeQuietly(connection); } } - private void truncate(DataSource dataSource, String table) throws SQLException { + public int getRowCount(BasicDataSource dataSource, String table) { + Connection connection = null; + Statement statement = null; + ResultSet resultSet = null; + try { + connection = dataSource.getConnection(); + statement = connection.createStatement(); + resultSet = statement.executeQuery("SELECT count(*) from " + table); + + return resultSet.next() ? resultSet.getInt(1) : 0; + } catch (SQLException e) { + throw new SonarException("Fail to get row count for table " + table, e); + } finally { + closeQuietly(resultSet); + closeQuietly(statement); + closeQuietly(connection); + } + } + + public DbTemplate truncate(DataSource dataSource, String table) { Connection connection = null; Statement statement = null; try { connection = dataSource.getConnection(); statement = connection.createStatement(); statement.executeUpdate("TRUNCATE TABLE " + table); + } catch (SQLException e) { + throw new SonarException("Fail to truncate table " + table, e); } finally { closeQuietly(statement); closeQuietly(connection); } + + return this; } - private BasicDataSource dataSource(String driver, String user, String password, String url) { + public BasicDataSource dataSource(String driver, String user, String password, String url) { BasicDataSource dataSource = new BasicDataSource(); dataSource.setDriverClassName(driver); dataSource.setUsername(user); @@ -155,16 +148,18 @@ public class LocalDatabaseFactory implements ServerComponent { return dataSource; } - public void create(DataSource dataSource, String dialect) throws SQLException { + public DbTemplate createSchema(DataSource dataSource, String dialect) { Connection connection = null; try { connection = dataSource.getConnection(); DdlUtils.createSchema(connection, dialect); } catch (SQLException e) { - throw new SonarException("Fail to create local database schema", e); + throw new SonarException("Fail to createSchema local database schema", e); } finally { closeQuietly(connection); } + + return this; } private void closeQuietly(Connection connection) { diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/LocalDatabaseFactory.java b/sonar-core/src/main/java/org/sonar/core/persistence/LocalDatabaseFactory.java new file mode 100644 index 00000000000..94d83bfaf5c --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/persistence/LocalDatabaseFactory.java @@ -0,0 +1,90 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.core.persistence; + +import com.google.common.io.Files; +import org.apache.commons.dbcp.BasicDataSource; +import org.sonar.api.ServerComponent; +import org.sonar.api.utils.SonarException; + +import javax.sql.DataSource; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; + +public class LocalDatabaseFactory implements ServerComponent { + private static final String DIALECT = "h2"; + private static final String DRIVER = "org.h2.Driver"; + private static final String URL = "jdbc:h2:"; + private static final String USER = "sonar"; + private static final String PASSWORD = "sonar"; + + private final Database database; + + public LocalDatabaseFactory(Database database) { + this.database = database; + } + + public byte[] createDatabaseForLocalMode() { + String name = System.getenv("java.io") + System.nanoTime(); // TODO + + try { + BasicDataSource destination = create(DIALECT, DRIVER, USER, PASSWORD, URL + name); + copy(database.getDataSource(), destination); + close(destination); + + return dbFileContent(name); + } catch (SQLException e) { + throw new SonarException("Unable to create database for local mode", e); + } + } + + private void copy(DataSource source, DataSource dest) { + new DbTemplate().copyTable(source, dest, "PROPERTIES", "SELECT * FROM PROPERTIES WHERE (USER_ID IS NULL) AND (RESOURCE_ID IS NULL) AND NOT (PROP_KEY LIKE '%.secured')") + .copyTable(source, dest, "RULES_PROFILES", "SELECT * FROM RULES_PROFILES") + .copyTable(source, dest, "RULES", "SELECT * FROM RULES") + .copyTable(source, dest, "RULES_PARAMETERS", "SELECT * FROM RULES_PARAMETERS") + .copyTable(source, dest, "ACTIVE_RULES", "SELECT * FROM ACTIVE_RULES") + .copyTable(source, dest, "ACTIVE_RULE_PARAMETERS", "SELECT * FROM ACTIVE_RULE_PARAMETERS") + .copyTable(source, dest, "METRICS", "SELECT * FROM METRICS"); + } + + private BasicDataSource create(String dialect, String driver, String user, String password, String url) { + BasicDataSource dataSource = new DbTemplate().dataSource(driver, user, password, url); + new DbTemplate().createSchema(dataSource, dialect); + return dataSource; + } + + private void close(BasicDataSource dest) throws SQLException { + dest.close(); + } + + private byte[] dbFileContent(String name) { + try { + File dbFile = new File(name + ".h2.db"); + byte[] content = Files.toByteArray(dbFile); + dbFile.delete(); + return content; + } catch (IOException e) { + throw new SonarException("Unable to read h2 database file", e); + } + } +} diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/LocalDatabaseFactoryTest.java b/sonar-core/src/test/java/org/sonar/core/persistence/LocalDatabaseFactoryTest.java new file mode 100644 index 00000000000..37f48eb2110 --- /dev/null +++ b/sonar-core/src/test/java/org/sonar/core/persistence/LocalDatabaseFactoryTest.java @@ -0,0 +1,75 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.core.persistence; + +import com.google.common.io.Files; +import org.apache.commons.dbcp.BasicDataSource; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; + +import static org.fest.assertions.Assertions.assertThat; + +public class LocalDatabaseFactoryTest extends AbstractDaoTestCase { + private LocalDatabaseFactory localDatabaseFactory; + private BasicDataSource dataSource; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Before + public void setUp() { + localDatabaseFactory = new LocalDatabaseFactory(getDatabase()); + } + + @After + public void closeDatabase() throws SQLException { + if (dataSource != null) { + dataSource.close(); + } + } + + @Test + public void should_create_database() throws IOException { + setupData("should_create_database"); + + byte[] database = localDatabaseFactory.createDatabaseForLocalMode(); + dataSource = createDatabase(database); + + assertThat(rowCount("PROPERTIES")).isEqualTo(1); + assertThat(rowCount("PROJECTS")).isZero(); + } + + private BasicDataSource createDatabase(byte[] db) throws IOException { + File file = temporaryFolder.newFile("db.h2.db"); + Files.write(db, file); + return new DbTemplate().dataSource("org.h2.Driver", "sonar", "sonar", "jdbc:h2:" + file.getAbsolutePath().replaceAll(".h2.db", "")); + } + + private int rowCount(String table) { + return new DbTemplate().getRowCount(dataSource, table); + } +} diff --git a/sonar-core/src/test/resources/org/sonar/core/persistence/LocalDatabaseFactoryTest/should_create_database.xml b/sonar-core/src/test/resources/org/sonar/core/persistence/LocalDatabaseFactoryTest/should_create_database.xml new file mode 100644 index 00000000000..281958780db --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/persistence/LocalDatabaseFactoryTest/should_create_database.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java index 88b9ae05c03..b5d69bd2890 100644 --- a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java +++ b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java @@ -68,7 +68,7 @@ import org.sonar.server.charts.ChartFactory; import org.sonar.server.configuration.Backup; import org.sonar.server.configuration.ProfilesManager; import org.sonar.server.database.EmbeddedDatabaseFactory; -import org.sonar.server.database.LocalDatabaseFactory; +import org.sonar.core.persistence.LocalDatabaseFactory; import org.sonar.server.notifications.NotificationService; import org.sonar.server.notifications.reviews.ReviewsNotificationManager; import org.sonar.server.plugins.ApplicationDeployer; diff --git a/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/synchro_controller.rb b/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/synchro_controller.rb index d3cd927e6bf..c9374031249 100644 --- a/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/synchro_controller.rb +++ b/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/synchro_controller.rb @@ -24,14 +24,10 @@ class Api::SynchroController < Api::ApiController # curl http://localhost:9000/api/synchro -v def index - database_factory = java_facade.getCoreComponentByClassname('org.sonar.server.database.LocalDatabaseFactory') + database_factory = java_facade.getCoreComponentByClassname('org.sonar.core.persistence.LocalDatabaseFactory') - path = database_factory.createDatabaseForLocalMode() + dbFileContent = database_factory.createDatabaseForLocalMode() - hash = {:path => path} - - respond_to do |format| - format.json { render :json => jsonp(hash) } - end + send_data String.from_java_bytes(dbFileContent) end end -- 2.39.5