From 6c6dc197d187b7db43fb019dbcaa8fa1b165bbcf Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Mon, 2 Sep 2013 17:29:22 +0200 Subject: [PATCH] SONAR-4602 Put dryRun DBs in a cache --- .../persistence/DryRunDatabaseFactory.java | 114 ++++++++++++++++-- .../DryRunDatabaseFactoryTest.java | 103 ++++++++++++++-- 2 files changed, 196 insertions(+), 21 deletions(-) diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java b/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java index e2da68aeeb3..b7ca73a233a 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java @@ -21,12 +21,17 @@ package org.sonar.core.persistence; import com.google.common.io.Files; import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFileFilter; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.ServerComponent; +import org.sonar.api.config.Settings; import org.sonar.api.issue.Issue; import org.sonar.api.platform.ServerFileSystem; import org.sonar.api.utils.SonarException; +import org.sonar.core.resource.ResourceDao; import javax.annotation.Nullable; import javax.sql.DataSource; @@ -34,8 +39,12 @@ import javax.sql.DataSource; import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.Collection; +import java.util.SortedSet; +import java.util.TreeSet; public class DryRunDatabaseFactory implements ServerComponent { + private static final String SONAR_DRY_RUN_CACHE_LAST_UPDATE = "sonar.dryRun.cache.lastUpdate"; private static final Logger LOG = LoggerFactory.getLogger(DryRunDatabaseFactory.class); private static final String DIALECT = "h2"; private static final String DRIVER = "org.h2.Driver"; @@ -47,25 +56,104 @@ public class DryRunDatabaseFactory implements ServerComponent { private final Database database; private final ServerFileSystem serverFileSystem; + private final Settings settings; + private final ResourceDao resourceDao; - public DryRunDatabaseFactory(Database database, ServerFileSystem serverFileSystem) { + public DryRunDatabaseFactory(Database database, ServerFileSystem serverFileSystem, Settings settings, ResourceDao resourceDao) { this.database = database; this.serverFileSystem = serverFileSystem; + this.settings = settings; + this.resourceDao = resourceDao; + } + + private File getRootCacheLocation() { + return new File(serverFileSystem.getTempDir(), "dryRun"); + } + + private File getCacheLocation(@Nullable Long projectId) { + return new File(getRootCacheLocation(), projectId != null ? projectId.toString() : "default"); + } + + private Long getLastTimestampInCache(@Nullable Long projectId) { + File cacheLocation = getCacheLocation(projectId); + if (!cacheLocation.exists()) { + return null; + } + Collection dbInCache = FileUtils.listFiles(cacheLocation, FileFileFilter.FILE, null); + if (dbInCache.isEmpty()) { + return null; + } + SortedSet timestamps = new TreeSet(); + for (File file : dbInCache) { + if (file.getName().endsWith(H2_FILE_SUFFIX)) { + try { + timestamps.add(Long.valueOf(StringUtils.removeEnd(file.getName(), H2_FILE_SUFFIX))); + } catch (NumberFormatException e) { + LOG.warn("Unexpected file in dryrun cache folder " + file.getAbsolutePath(), e); + } + } + } + if (timestamps.isEmpty()) { + return null; + } + return timestamps.last(); + } + + private boolean isValid(@Nullable Long projectId, long lastTimestampInCache) { + long globalTimestamp = settings.getLong(SONAR_DRY_RUN_CACHE_LAST_UPDATE); + if (globalTimestamp > lastTimestampInCache) { + return false; + } + if (projectId != null) { + // For modules look for root project last modification timestamp + Long rootId = resourceDao.getRootProjectByComponentId(projectId).getId(); + long projectTimestamp = settings.getLong("sonar.dryRun.cache." + rootId + ".lastUpdate"); + if (projectTimestamp > lastTimestampInCache) { + return false; + } + } + return true; } public byte[] createDatabaseForDryRun(@Nullable Long projectId) { long startup = System.currentTimeMillis(); - String name = serverFileSystem.getTempDir().getAbsolutePath() + "db-" + System.nanoTime(); + + Long lastTimestampInCache = getLastTimestampInCache(projectId); + if (lastTimestampInCache == null || !isValid(projectId, lastTimestampInCache)) { + lastTimestampInCache = System.nanoTime(); + cleanCache(projectId); + createNewDatabaseForDryRun(projectId, startup, lastTimestampInCache); + } + return dbFileContent(projectId, lastTimestampInCache); + } + + private void cleanCache(@Nullable Long projectId) { + FileUtils.deleteQuietly(getCacheLocation(projectId)); + } + + public String getH2DBName(File location, long timestamp) { + return location.getAbsolutePath() + File.separator + timestamp; + } + + public String getTemporaryH2DBName(File location, long timestamp) { + return location.getAbsolutePath() + File.separator + ".tmp" + timestamp; + } + + private void createNewDatabaseForDryRun(Long projectId, long startup, Long lastTimestampInCache) { + String tmpName = getTemporaryH2DBName(getCacheLocation(projectId), lastTimestampInCache); + String finalName = getH2DBName(getCacheLocation(projectId), lastTimestampInCache); try { DataSource source = database.getDataSource(); - BasicDataSource destination = create(DIALECT, DRIVER, USER, PASSWORD, URL + name); + BasicDataSource destination = create(DIALECT, DRIVER, USER, PASSWORD, URL + tmpName); copy(source, destination, projectId); close(destination); + File tempDbFile = new File(tmpName + H2_FILE_SUFFIX); + File dbFile = new File(finalName + H2_FILE_SUFFIX); + Files.move(tempDbFile, dbFile); if (LOG.isDebugEnabled()) { - File dbFile = new File(name + H2_FILE_SUFFIX); long size = dbFile.length(); long duration = System.currentTimeMillis() - startup; if (projectId == null) { @@ -75,10 +163,12 @@ public class DryRunDatabaseFactory implements ServerComponent { } } - return dbFileContent(name); } catch (SQLException e) { throw new SonarException("Unable to create database for DryRun", e); + } catch (IOException e) { + throw new SonarException("Unable to cache database for DryRun", e); } + } private void copy(DataSource source, DataSource dest, @Nullable Long projectId) { @@ -137,14 +227,20 @@ public class DryRunDatabaseFactory implements ServerComponent { destination.close(); } - private byte[] dbFileContent(String name) { + private byte[] dbFileContent(@Nullable Long projectId, long timestamp) { + File cacheLocation = getCacheLocation(projectId); + try { + FileUtils.forceMkdir(cacheLocation); + } catch (IOException e) { + throw new SonarException("Unable to create cache directory " + cacheLocation, e); + } + String name = getH2DBName(cacheLocation, timestamp); try { File dbFile = new File(name + H2_FILE_SUFFIX); - byte[] content = Files.toByteArray(dbFile); - dbFile.delete(); - return content; + return Files.toByteArray(dbFile); } catch (IOException e) { throw new SonarException("Unable to read h2 database file", e); } } + } diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java b/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java index ce9f35f52fc..bb5c335a4ca 100644 --- a/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java +++ b/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java @@ -19,14 +19,19 @@ */ package org.sonar.core.persistence; +import com.google.common.base.Charsets; import com.google.common.io.Files; import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.Settings; import org.sonar.api.platform.ServerFileSystem; +import org.sonar.core.resource.ResourceDao; +import org.sonar.core.resource.ResourceDto; import java.io.File; import java.io.IOException; @@ -43,10 +48,18 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private File dryRunCache; + private ResourceDao resourceDao; + private Settings settings; @Before - public void setUp() { - localDatabaseFactory = new DryRunDatabaseFactory(getDatabase(), serverFileSystem); + public void setUp() throws Exception { + File tempFolder = temporaryFolder.newFolder(); + dryRunCache = new File(tempFolder, "dryRun"); + when(serverFileSystem.getTempDir()).thenReturn(tempFolder); + resourceDao = mock(ResourceDao.class); + settings = new Settings(); + localDatabaseFactory = new DryRunDatabaseFactory(getDatabase(), serverFileSystem, settings, resourceDao); } @After @@ -60,7 +73,7 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { public void should_create_database_without_project() throws IOException, SQLException { setupData("should_create_database"); - when(serverFileSystem.getTempDir()).thenReturn(temporaryFolder.newFolder()); + assertThat(new File(dryRunCache, "default")).doesNotExist(); byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); dataSource = createDatabase(database); @@ -68,13 +81,84 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { assertThat(rowCount("metrics")).isEqualTo(2); assertThat(rowCount("projects")).isZero(); assertThat(rowCount("alerts")).isEqualTo(1); + + assertThat(new File(dryRunCache, "default")).isDirectory(); + } + + @Test + public void should_reuse_database_without_project() throws IOException, SQLException { + setupData("should_create_database"); + + FileUtils.write(new File(new File(dryRunCache, "default"), "123456.h2.db"), "fakeDbContent"); + + byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); + + assertThat(new String(database, Charsets.UTF_8)).isEqualTo("fakeDbContent"); + } + + @Test + public void should_evict_database_without_project() throws IOException, SQLException { + setupData("should_create_database"); + + // There is a DB in cache + File existingDb = new File(new File(dryRunCache, "default"), "123456.h2.db"); + FileUtils.write(existingDb, "fakeDbContent"); + + // But last modification timestamp is greater + settings.setProperty("sonar.dryRun.cache.lastUpdate", "123457"); + + byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); + dataSource = createDatabase(database); + + assertThat(rowCount("metrics")).isEqualTo(2); + assertThat(rowCount("projects")).isZero(); + assertThat(rowCount("alerts")).isEqualTo(1); + + // Previous cached DB was deleted + assertThat(existingDb).doesNotExist(); } @Test public void should_create_database_with_project() throws IOException, SQLException { setupData("should_create_database"); - when(serverFileSystem.getTempDir()).thenReturn(temporaryFolder.newFolder()); + assertThat(new File(dryRunCache, "123")).doesNotExist(); + + byte[] database = localDatabaseFactory.createDatabaseForDryRun(123L); + dataSource = createDatabase(database); + + assertThat(rowCount("metrics")).isEqualTo(2); + assertThat(rowCount("projects")).isEqualTo(1); + assertThat(rowCount("snapshots")).isEqualTo(1); + assertThat(rowCount("project_measures")).isEqualTo(1); + + assertThat(new File(dryRunCache, "123")).isDirectory(); + } + + @Test + public void should_reuse_database_with_project() throws IOException, SQLException { + setupData("should_create_database"); + + FileUtils.write(new File(new File(dryRunCache, "123"), "123456.h2.db"), "fakeDbContent"); + + when(resourceDao.getRootProjectByComponentId(123L)).thenReturn(new ResourceDto().setId(123L)); + byte[] database = localDatabaseFactory.createDatabaseForDryRun(123L); + + assertThat(new String(database, Charsets.UTF_8)).isEqualTo("fakeDbContent"); + } + + @Test + public void should_evict_database_with_project() throws IOException, SQLException { + setupData("should_create_database"); + + when(resourceDao.getRootProjectByComponentId(123L)).thenReturn(new ResourceDto().setId(123L)); + + // There is a DB in cache + File existingDb = new File(new File(dryRunCache, "123"), "123456.h2.db"); + FileUtils.write(existingDb, "fakeDbContent"); + + // But last project modification timestamp is greater + settings.setProperty("sonar.dryRun.cache.123.lastUpdate", "123457"); byte[] database = localDatabaseFactory.createDatabaseForDryRun(123L); dataSource = createDatabase(database); @@ -83,14 +167,15 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { assertThat(rowCount("projects")).isEqualTo(1); assertThat(rowCount("snapshots")).isEqualTo(1); assertThat(rowCount("project_measures")).isEqualTo(1); + + // Previous cached DB was deleted + assertThat(existingDb).doesNotExist(); } @Test public void should_create_database_with_issues() throws IOException, SQLException { setupData("should_create_database_with_issues"); - when(serverFileSystem.getTempDir()).thenReturn(temporaryFolder.newFolder()); - byte[] database = localDatabaseFactory.createDatabaseForDryRun(399L); dataSource = createDatabase(database); @@ -116,8 +201,6 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { public void should_export_issues_of_sub_module() throws IOException, SQLException { setupData("multi-modules-with-issues"); - when(serverFileSystem.getTempDir()).thenReturn(temporaryFolder.newFolder()); - // 301 : sub module with 1 closed issue and 1 open issue byte[] database = localDatabaseFactory.createDatabaseForDryRun(301L); dataSource = createDatabase(database); @@ -131,8 +214,6 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { public void should_export_issues_of_sub_module_2() throws IOException, SQLException { setupData("multi-modules-with-issues"); - when(serverFileSystem.getTempDir()).thenReturn(temporaryFolder.newFolder()); - // 302 : sub module without any issues byte[] database = localDatabaseFactory.createDatabaseForDryRun(302L); dataSource = createDatabase(database); @@ -143,8 +224,6 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { public void should_copy_permission_templates_data() throws Exception { setupData("should_copy_permission_templates"); - when(serverFileSystem.getTempDir()).thenReturn(temporaryFolder.newFolder()); - byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); dataSource = createDatabase(database); assertThat(rowCount("permission_templates")).isEqualTo(1); -- 2.39.5