From cdfcc9f9bd4e8e5caf7a656ed2cafd20b6b98c4b Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Fri, 6 Sep 2013 11:36:56 +0200 Subject: [PATCH] SONAR-4602 Fix concurrent access issues with dryRun cache --- .../sonar/batch/phases/UpdateStatusJob.java | 31 ++-- .../batch/phases/UpdateStatusJobTest.java | 35 +++- .../org/sonar/core/dryrun/DryRunCache.java | 119 ++++++++++++-- .../persistence/DryRunDatabaseFactory.java | 102 +----------- .../sonar/core/dryrun/DryRunCacheTest.java | 153 ++++++++++++++---- .../DryRunDatabaseFactoryTest.java | 107 ++---------- .../java/org/sonar/server/ui/JRubyFacade.java | 4 +- .../app/controllers/alerts_controller.rb | 2 +- .../controllers/batch_bootstrap_controller.rb | 16 ++ .../app/controllers/project_controller.rb | 2 +- 10 files changed, 318 insertions(+), 253 deletions(-) diff --git a/sonar-batch/src/main/java/org/sonar/batch/phases/UpdateStatusJob.java b/sonar-batch/src/main/java/org/sonar/batch/phases/UpdateStatusJob.java index 3bd4a77946a..dc338ddd360 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/phases/UpdateStatusJob.java +++ b/sonar-batch/src/main/java/org/sonar/batch/phases/UpdateStatusJob.java @@ -29,11 +29,9 @@ import org.sonar.api.database.DatabaseSession; import org.sonar.api.database.model.Snapshot; import org.sonar.api.resources.Project; import org.sonar.api.resources.Scopes; +import org.sonar.api.utils.SonarException; import org.sonar.batch.bootstrap.ServerClient; import org.sonar.batch.index.ResourcePersister; -import org.sonar.core.dryrun.DryRunCache; -import org.sonar.core.properties.PropertiesDao; -import org.sonar.core.properties.PropertyDto; import javax.persistence.Query; @@ -41,6 +39,8 @@ import java.util.List; public class UpdateStatusJob implements BatchComponent { + private static final Logger LOG = LoggerFactory.getLogger(UpdateStatusJob.class); + private DatabaseSession session; private ServerClient server; // TODO remove this component @@ -48,31 +48,36 @@ public class UpdateStatusJob implements BatchComponent { private ResourcePersister resourcePersister; private Settings settings; private Project project; - private PropertiesDao propertiesDao; public UpdateStatusJob(Settings settings, ServerClient server, DatabaseSession session, - ResourcePersister resourcePersister, Project project, Snapshot snapshot, PropertiesDao propertiesDao) { + ResourcePersister resourcePersister, Project project, Snapshot snapshot) { this.session = session; this.server = server; this.resourcePersister = resourcePersister; this.project = project; this.snapshot = snapshot; this.settings = settings; - this.propertiesDao = propertiesDao; } public void execute() { disablePreviousSnapshot(); enableCurrentSnapshot(); - updateDryRunLastModificationTimestamp(); + evictDryRunDB(); } - private void updateDryRunLastModificationTimestamp() { - propertiesDao.setProperty( - new PropertyDto() - .setKey(DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY) - .setResourceId(Long.valueOf(project.getId())) - .setValue(String.valueOf(System.nanoTime()))); + @VisibleForTesting + void evictDryRunDB() { + if (settings.getBoolean(CoreProperties.DRY_RUN)) { + // If this is a dryRun analysis then we should not evict dryRun database + return; + } + String url = "/batch_bootstrap/evict?project=" + project.getId(); + try { + LOG.debug("Evict dryRun database"); + server.request(url); + } catch (Exception e) { + throw new SonarException("Unable to evict dryRun database: " + url, e); + } } private void disablePreviousSnapshot() { diff --git a/sonar-batch/src/test/java/org/sonar/batch/phases/UpdateStatusJobTest.java b/sonar-batch/src/test/java/org/sonar/batch/phases/UpdateStatusJobTest.java index 6b4040d15b5..fd975e034de 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/phases/UpdateStatusJobTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/phases/UpdateStatusJobTest.java @@ -32,12 +32,14 @@ import org.sonar.batch.index.DefaultResourcePersister; import org.sonar.batch.index.ResourceCache; import org.sonar.batch.index.ResourcePersister; import org.sonar.batch.index.SnapshotCache; -import org.sonar.core.properties.PropertiesDao; import org.sonar.jpa.test.AbstractDbUnitTestCase; import javax.persistence.Query; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.contains; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; public class UpdateStatusJobTest extends AbstractDbUnitTestCase { @@ -65,7 +67,7 @@ public class UpdateStatusJobTest extends AbstractDbUnitTestCase { project.setId(1); UpdateStatusJob job = new UpdateStatusJob(new Settings().appendProperty(CoreProperties.SERVER_BASE_URL, "http://myserver/"), mock(ServerClient.class), session, new DefaultResourcePersister(session, mock(ResourcePermissions.class), mock(SnapshotCache.class), mock(ResourceCache.class)), - project, loadSnapshot(snapshotId), mock(PropertiesDao.class)); + project, loadSnapshot(snapshotId)); job.execute(); checkTables(fixture, "snapshots"); @@ -83,7 +85,7 @@ public class UpdateStatusJobTest extends AbstractDbUnitTestCase { settings.setProperty(CoreProperties.SERVER_BASE_URL, "http://myserver/"); Project project = new Project("struts"); UpdateStatusJob job = new UpdateStatusJob(settings, mock(ServerClient.class), mock(DatabaseSession.class), - mock(ResourcePersister.class), project, mock(Snapshot.class), mock(PropertiesDao.class)); + mock(ResourcePersister.class), project, mock(Snapshot.class)); Logger logger = mock(Logger.class); job.logSuccess(logger); @@ -97,11 +99,36 @@ public class UpdateStatusJobTest extends AbstractDbUnitTestCase { settings.setProperty("sonar.dryRun", true); Project project = new Project("struts"); UpdateStatusJob job = new UpdateStatusJob(settings, mock(ServerClient.class), mock(DatabaseSession.class), - mock(ResourcePersister.class), project, mock(Snapshot.class), mock(PropertiesDao.class)); + mock(ResourcePersister.class), project, mock(Snapshot.class)); Logger logger = mock(Logger.class); job.logSuccess(logger); verify(logger).info("ANALYSIS SUCCESSFUL"); } + + @Test + public void should_evict_cache_for_regular_analysis() throws Exception { + Settings settings = new Settings(); + Project project = new Project("struts"); + ServerClient serverClient = mock(ServerClient.class); + UpdateStatusJob job = new UpdateStatusJob(settings, serverClient, mock(DatabaseSession.class), + mock(ResourcePersister.class), project, mock(Snapshot.class)); + + job.evictDryRunDB(); + verify(serverClient).request(contains("/batch_bootstrap/evict")); + } + + @Test + public void should_not_evict_cache_for_dry_run_analysis() throws Exception { + Settings settings = new Settings(); + settings.setProperty("sonar.dryRun", true); + Project project = new Project("struts"); + ServerClient serverClient = mock(ServerClient.class); + UpdateStatusJob job = new UpdateStatusJob(settings, serverClient, mock(DatabaseSession.class), + mock(ResourcePersister.class), project, mock(Snapshot.class)); + + job.evictDryRunDB(); + verify(serverClient, never()).request(anyString()); + } } diff --git a/sonar-core/src/main/java/org/sonar/core/dryrun/DryRunCache.java b/sonar-core/src/main/java/org/sonar/core/dryrun/DryRunCache.java index b4b7ffedb8c..d92ce5ccd94 100644 --- a/sonar-core/src/main/java/org/sonar/core/dryrun/DryRunCache.java +++ b/sonar-core/src/main/java/org/sonar/core/dryrun/DryRunCache.java @@ -19,10 +19,14 @@ */ package org.sonar.core.dryrun; +import com.google.common.io.Files; import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.ServerExtension; import org.sonar.api.platform.ServerFileSystem; import org.sonar.api.utils.SonarException; +import org.sonar.core.persistence.DryRunDatabaseFactory; import org.sonar.core.properties.PropertiesDao; import org.sonar.core.properties.PropertyDto; import org.sonar.core.resource.ResourceDao; @@ -31,22 +35,107 @@ import org.sonar.core.resource.ResourceDto; import javax.annotation.Nullable; import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * @since 4.0 */ public class DryRunCache implements ServerExtension { + private static final Logger LOG = LoggerFactory.getLogger(DryRunCache.class); + public static final String SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY = "sonar.dryRun.cache.lastUpdate"; private ServerFileSystem serverFileSystem; private PropertiesDao propertiesDao; private ResourceDao resourceDao; - public DryRunCache(ServerFileSystem serverFileSystem, PropertiesDao propertiesDao, ResourceDao resourceDao) { + private Map lockPerProject = new HashMap(); + private Map lastTimestampPerProject = new HashMap(); + + private DryRunDatabaseFactory dryRunDatabaseFactory; + + public DryRunCache(ServerFileSystem serverFileSystem, PropertiesDao propertiesDao, ResourceDao resourceDao, DryRunDatabaseFactory dryRunDatabaseFactory) { this.serverFileSystem = serverFileSystem; this.propertiesDao = propertiesDao; this.resourceDao = resourceDao; + this.dryRunDatabaseFactory = dryRunDatabaseFactory; + } + + public byte[] getDatabaseForDryRun(@Nullable Long projectId) { + long notNullProjectId = projectId != null ? projectId.longValue() : 0L; + ReadWriteLock rwl = getLock(notNullProjectId); + try { + rwl.readLock().lock(); + if (!isCacheValid(projectId)) { + // upgrade lock manually + // must unlock first to obtain writelock + rwl.readLock().unlock(); + rwl.writeLock().lock(); + // recheck + if (!isCacheValid(projectId)) { + generateNewDB(projectId); + } + // downgrade lock + // reacquire read without giving up write lock + rwl.readLock().lock(); + // unlock write, still hold read + rwl.writeLock().unlock(); + } + File dbFile = new File(getCacheLocation(projectId), lastTimestampPerProject.get(notNullProjectId) + DryRunDatabaseFactory.H2_FILE_SUFFIX); + return fileToByte(dbFile); + } finally { + rwl.readLock().unlock(); + } + } + + private boolean isCacheValid(@Nullable Long projectId) { + long notNullProjectId = projectId != null ? projectId.longValue() : 0L; + Long lastTimestampInCache = lastTimestampPerProject.get(notNullProjectId); + LOG.debug("Timestamp of last cached DB is {}", lastTimestampInCache); + if (lastTimestampInCache != null && isValid(projectId, lastTimestampInCache.longValue())) { + File dbFile = new File(getCacheLocation(projectId), lastTimestampInCache + DryRunDatabaseFactory.H2_FILE_SUFFIX); + LOG.debug("Look for existence of cached DB at {}", dbFile); + if (dbFile.exists()) { + LOG.debug("Found cached DB at {}", dbFile); + return true; + } + } + return false; + } + + private void generateNewDB(@Nullable Long projectId) { + if (projectId != null) { + LOG.info("Generate new dryRun database for project [id={}]", projectId); + } else { + LOG.info("Generate new dryRun database for new project"); + } + long notNullProjectId = projectId != null ? projectId.longValue() : 0L; + long newTimestamp = System.currentTimeMillis(); + File cacheLocation = getCacheLocation(projectId); + FileUtils.deleteQuietly(cacheLocation); + File dbFile = dryRunDatabaseFactory.createNewDatabaseForDryRun(projectId, cacheLocation, String.valueOf(newTimestamp)); + LOG.info("Cached DB at {}", dbFile); + lastTimestampPerProject.put(notNullProjectId, newTimestamp); + } + + private byte[] fileToByte(File dbFile) { + try { + return Files.toByteArray(dbFile); + } catch (IOException e) { + throw new SonarException("Unable to create h2 database file", e); + } + } + + private synchronized ReadWriteLock getLock(long notNullProjectId) { + if (!lockPerProject.containsKey(notNullProjectId)) { + lockPerProject.put(notNullProjectId, new ReentrantReadWriteLock(true)); + } + return lockPerProject.get(notNullProjectId); } private File getRootCacheLocation() { @@ -57,7 +146,21 @@ public class DryRunCache implements ServerExtension { return new File(getRootCacheLocation(), projectId != null ? projectId.toString() : "default"); } - public long getModificationTimestamp(@Nullable Long projectId) { + private boolean isValid(@Nullable Long projectId, long lastTimestampInCache) { + long globalTimestamp = getModificationTimestamp(null); + if (globalTimestamp > lastTimestampInCache) { + return false; + } + if (projectId != null) { + long projectTimestamp = getModificationTimestamp(projectId); + if (projectTimestamp > lastTimestampInCache) { + return false; + } + } + return true; + } + + private long getModificationTimestamp(@Nullable Long projectId) { if (projectId == null) { PropertyDto dto = propertiesDao.selectGlobalProperty(SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY); if (dto == null) { @@ -84,16 +187,8 @@ public class DryRunCache implements ServerExtension { propertiesDao.deleteAllProperties(SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY); } - public void clean(@Nullable Long resourceId) { - // Delete folder where dryRun DB are stored - FileUtils.deleteQuietly(getCacheLocation(resourceId)); - } - public void reportGlobalModification() { - // Delete folder where dryRun DB are stored - FileUtils.deleteQuietly(getRootCacheLocation()); - - propertiesDao.setProperty(new PropertyDto().setKey(SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY).setValue(String.valueOf(System.nanoTime()))); + propertiesDao.setProperty(new PropertyDto().setKey(SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY).setValue(String.valueOf(System.currentTimeMillis()))); } public void reportResourceModification(String resourceKey) { @@ -102,6 +197,6 @@ public class DryRunCache implements ServerExtension { throw new SonarException("Unable to find root project for component with [key=" + resourceKey + "]"); } propertiesDao.setProperty(new PropertyDto().setKey(SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY).setResourceId(rootProject.getId()) - .setValue(String.valueOf(System.nanoTime()))); + .setValue(String.valueOf(System.currentTimeMillis()))); } } diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java b/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java index 7369d11e8d0..1d52f4dd296 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DryRunDatabaseFactory.java @@ -19,119 +19,48 @@ */ package org.sonar.core.persistence; -import com.google.common.io.Files; import org.apache.commons.dbcp.BasicDataSource; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FileFileFilter; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.ServerComponent; import org.sonar.api.issue.Issue; import org.sonar.api.utils.SonarException; -import org.sonar.core.dryrun.DryRunCache; import javax.annotation.Nullable; import javax.sql.DataSource; import java.io.File; -import java.io.IOException; import java.sql.SQLException; -import java.util.Collection; -import java.util.SortedSet; -import java.util.TreeSet; public class DryRunDatabaseFactory implements ServerComponent { private static final Logger LOG = LoggerFactory.getLogger(DryRunDatabaseFactory.class); private static final String DIALECT = "h2"; private static final String DRIVER = "org.h2.Driver"; private static final String URL = "jdbc:h2:"; - private static final String H2_FILE_SUFFIX = ".h2.db"; + public static final String H2_FILE_SUFFIX = ".h2.db"; private static final String SONAR = "sonar"; private static final String USER = SONAR; private static final String PASSWORD = SONAR; private final Database database; - private DryRunCache dryRunCache; - public DryRunDatabaseFactory(Database database, DryRunCache dryRunCache) { + public DryRunDatabaseFactory(Database database) { this.database = database; - this.dryRunCache = dryRunCache; } - private Long getLastTimestampInCache(@Nullable Long projectId) { - File cacheLocation = dryRunCache.getCacheLocation(projectId); - if (!cacheLocation.exists()) { - return null; - } - Collection dbInCache = FileUtils.listFiles(cacheLocation, FileFileFilter.FILE, null); - if (dbInCache.isEmpty()) { - return null; - } - SortedSet timestamps = new TreeSet(); - for (File file : dbInCache) { - if (file.getName().endsWith(H2_FILE_SUFFIX)) { - try { - timestamps.add(Long.valueOf(StringUtils.removeEnd(file.getName(), H2_FILE_SUFFIX))); - } catch (NumberFormatException e) { - LOG.warn("Unexpected file in dryrun cache folder " + file.getAbsolutePath(), e); - } - } - } - if (timestamps.isEmpty()) { - return null; - } - return timestamps.last(); - } - - private boolean isValid(@Nullable Long projectId, long lastTimestampInCache) { - long globalTimestamp = dryRunCache.getModificationTimestamp(null); - if (globalTimestamp > lastTimestampInCache) { - return false; - } - if (projectId != null) { - long projectTimestamp = dryRunCache.getModificationTimestamp(projectId); - if (projectTimestamp > lastTimestampInCache) { - return false; - } - } - return true; - } - - public byte[] createDatabaseForDryRun(@Nullable Long projectId) { + public File createNewDatabaseForDryRun(Long projectId, File destFolder, String dbFileName) { long startup = System.currentTimeMillis(); - Long lastTimestampInCache = getLastTimestampInCache(projectId); - if (lastTimestampInCache == null || !isValid(projectId, lastTimestampInCache)) { - lastTimestampInCache = System.nanoTime(); - dryRunCache.clean(projectId); - createNewDatabaseForDryRun(projectId, startup, lastTimestampInCache); - } - return dbFileContent(projectId, lastTimestampInCache); - } - - public String getH2DBName(File location, long timestamp) { - return location.getAbsolutePath() + File.separator + timestamp; - } - - public String getTemporaryH2DBName(File location, long timestamp) { - return location.getAbsolutePath() + File.separator + ".tmp" + timestamp; - } - - private void createNewDatabaseForDryRun(Long projectId, long startup, Long lastTimestampInCache) { - String tmpName = getTemporaryH2DBName(dryRunCache.getCacheLocation(projectId), lastTimestampInCache); - String finalName = getH2DBName(dryRunCache.getCacheLocation(projectId), lastTimestampInCache); + String h2Name = destFolder.getAbsolutePath() + File.separator + dbFileName; try { DataSource source = database.getDataSource(); - BasicDataSource destination = create(DIALECT, DRIVER, USER, PASSWORD, URL + tmpName); + BasicDataSource destination = create(DIALECT, DRIVER, USER, PASSWORD, URL + h2Name); copy(source, destination, projectId); close(destination); - File tempDbFile = new File(tmpName + H2_FILE_SUFFIX); - File dbFile = new File(finalName + H2_FILE_SUFFIX); - Files.move(tempDbFile, dbFile); + File dbFile = new File(h2Name + H2_FILE_SUFFIX); if (LOG.isDebugEnabled()) { long size = dbFile.length(); long duration = System.currentTimeMillis() - startup; @@ -141,11 +70,10 @@ public class DryRunDatabaseFactory implements ServerComponent { LOG.debug("Dry Run Database for project " + projectId + " created in " + duration + " ms, size is " + size + " bytes"); } } + return dbFile; } catch (SQLException e) { throw new SonarException("Unable to create database for DryRun", e); - } catch (IOException e) { - throw new SonarException("Unable to cache database for DryRun", e); } } @@ -209,20 +137,4 @@ public class DryRunDatabaseFactory implements ServerComponent { destination.close(); } - private byte[] dbFileContent(@Nullable Long projectId, long timestamp) { - File cacheLocation = dryRunCache.getCacheLocation(projectId); - try { - FileUtils.forceMkdir(cacheLocation); - } catch (IOException e) { - throw new SonarException("Unable to create cache directory " + cacheLocation, e); - } - String name = getH2DBName(cacheLocation, timestamp); - try { - File dbFile = new File(name + H2_FILE_SUFFIX); - return Files.toByteArray(dbFile); - } catch (IOException e) { - throw new SonarException("Unable to read h2 database file", e); - } - } - } diff --git a/sonar-core/src/test/java/org/sonar/core/dryrun/DryRunCacheTest.java b/sonar-core/src/test/java/org/sonar/core/dryrun/DryRunCacheTest.java index 6b9b808608c..935cc2a41aa 100644 --- a/sonar-core/src/test/java/org/sonar/core/dryrun/DryRunCacheTest.java +++ b/sonar-core/src/test/java/org/sonar/core/dryrun/DryRunCacheTest.java @@ -24,17 +24,26 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.sonar.api.platform.ServerFileSystem; +import org.sonar.core.persistence.DryRunDatabaseFactory; import org.sonar.core.properties.PropertiesDao; import org.sonar.core.properties.PropertyDto; import org.sonar.core.resource.ResourceDao; import org.sonar.core.resource.ResourceDto; import java.io.File; +import java.io.IOException; import static org.fest.assertions.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.isNull; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -48,61 +57,147 @@ public class DryRunCacheTest { private PropertiesDao propertiesDao; private ResourceDao resourceDao; + private DryRunDatabaseFactory dryRunDatabaseFactory; + + private File dryRunCacheLocation; + @Before - public void prepare() { + public void prepare() throws IOException { serverFileSystem = mock(ServerFileSystem.class); propertiesDao = mock(PropertiesDao.class); resourceDao = mock(ResourceDao.class); - dryRunCache = new DryRunCache(serverFileSystem, propertiesDao, resourceDao); - } + dryRunDatabaseFactory = mock(DryRunDatabaseFactory.class); - @Test - public void test_get_cache_location() throws Exception { - File tempFolder = temp.newFolder(); - when(serverFileSystem.getTempDir()).thenReturn(tempFolder); + File tempLocation = temp.newFolder(); + when(serverFileSystem.getTempDir()).thenReturn(tempLocation); + dryRunCacheLocation = new File(tempLocation, "dryRun"); - assertThat(dryRunCache.getCacheLocation(null)).isEqualTo(new File(new File(tempFolder, "dryRun"), "default")); - assertThat(dryRunCache.getCacheLocation(123L)).isEqualTo(new File(new File(tempFolder, "dryRun"), "123")); + dryRunCache = new DryRunCache(serverFileSystem, propertiesDao, resourceDao, dryRunDatabaseFactory); } @Test - public void test_get_modification_timestamp() { - when(propertiesDao.selectGlobalProperty(DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY)).thenReturn(null); - assertThat(dryRunCache.getModificationTimestamp(null)).isEqualTo(0L); - - when(propertiesDao.selectGlobalProperty(DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY)).thenReturn(new PropertyDto().setValue("123456")); - assertThat(dryRunCache.getModificationTimestamp(null)).isEqualTo(123456L); + public void test_getDatabaseForDryRun_on_new_project() throws Exception { + when(dryRunDatabaseFactory.createNewDatabaseForDryRun(isNull(Long.class), any(File.class), anyString())).thenAnswer(new Answer() { + public File answer(InvocationOnMock invocation) throws IOException { + Object[] args = invocation.getArguments(); + File dbFile = new File(new File(dryRunCacheLocation, "default"), (String) args[2] + ".h2.db"); + FileUtils.write(dbFile, "fake db content"); + return dbFile; + } + }); + byte[] dbContent = dryRunCache.getDatabaseForDryRun(null); + assertThat(new String(dbContent)).isEqualTo("fake db content"); + + dbContent = dryRunCache.getDatabaseForDryRun(null); + assertThat(new String(dbContent)).isEqualTo("fake db content"); + + verify(dryRunDatabaseFactory, times(1)).createNewDatabaseForDryRun(anyLong(), any(File.class), anyString()); + } - when(resourceDao.getRootProjectByComponentId(123L)).thenReturn(new ResourceDto().setId(456L)); + @Test + public void test_getDatabaseForDryRun_on_existing_project() throws Exception { + when(dryRunDatabaseFactory.createNewDatabaseForDryRun(eq(123L), any(File.class), anyString())).thenAnswer(new Answer() { + public File answer(InvocationOnMock invocation) throws IOException { + Object[] args = invocation.getArguments(); + File dbFile = new File(new File(dryRunCacheLocation, "123"), (String) args[2] + ".h2.db"); + FileUtils.write(dbFile, "fake db content"); + return dbFile; + } + }); + when(resourceDao.getRootProjectByComponentId(123L)).thenReturn(new ResourceDto().setId(123L)); + byte[] dbContent = dryRunCache.getDatabaseForDryRun(123L); + assertThat(new String(dbContent)).isEqualTo("fake db content"); + + dbContent = dryRunCache.getDatabaseForDryRun(123L); + assertThat(new String(dbContent)).isEqualTo("fake db content"); + + verify(dryRunDatabaseFactory, times(1)).createNewDatabaseForDryRun(anyLong(), any(File.class), anyString()); + } - when(propertiesDao.selectProjectProperty(456L, DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY)).thenReturn(null); - assertThat(dryRunCache.getModificationTimestamp(123L)).isEqualTo(0L); + @Test + public void test_getDatabaseForDryRun_global_invalidation() throws Exception { + when(dryRunDatabaseFactory.createNewDatabaseForDryRun(isNull(Long.class), any(File.class), anyString())) + .thenAnswer(new Answer() { + public File answer(InvocationOnMock invocation) throws IOException { + Object[] args = invocation.getArguments(); + File dbFile = new File(new File(dryRunCacheLocation, "default"), (String) args[2] + ".h2.db"); + FileUtils.write(dbFile, "fake db content 1"); + return dbFile; + } + }) + .thenAnswer(new Answer() { + public File answer(InvocationOnMock invocation) throws IOException { + Object[] args = invocation.getArguments(); + File dbFile = new File(new File(dryRunCacheLocation, "default"), (String) args[2] + ".h2.db"); + FileUtils.write(dbFile, "fake db content 2"); + return dbFile; + } + }); + byte[] dbContent = dryRunCache.getDatabaseForDryRun(null); + assertThat(new String(dbContent)).isEqualTo("fake db content 1"); + + // Emulate invalidation of cache + Thread.sleep(100); + when(propertiesDao.selectGlobalProperty(DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY)).thenReturn(new PropertyDto().setValue("" + System.currentTimeMillis())); + + dbContent = dryRunCache.getDatabaseForDryRun(null); + assertThat(new String(dbContent)).isEqualTo("fake db content 2"); + + verify(dryRunDatabaseFactory, times(2)).createNewDatabaseForDryRun(anyLong(), any(File.class), anyString()); + } - when(propertiesDao.selectProjectProperty(456L, DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY)).thenReturn(new PropertyDto().setValue("123456")); - assertThat(dryRunCache.getModificationTimestamp(123L)).isEqualTo(123456L); + @Test + public void test_getDatabaseForDryRun_project_invalidation() throws Exception { + when(dryRunDatabaseFactory.createNewDatabaseForDryRun(eq(123L), any(File.class), anyString())) + .thenAnswer(new Answer() { + public File answer(InvocationOnMock invocation) throws IOException { + Object[] args = invocation.getArguments(); + File dbFile = new File(new File(dryRunCacheLocation, "123"), (String) args[2] + ".h2.db"); + FileUtils.write(dbFile, "fake db content 1"); + return dbFile; + } + }) + .thenAnswer(new Answer() { + public File answer(InvocationOnMock invocation) throws IOException { + Object[] args = invocation.getArguments(); + File dbFile = new File(new File(dryRunCacheLocation, "123"), (String) args[2] + ".h2.db"); + FileUtils.write(dbFile, "fake db content 2"); + return dbFile; + } + }); + when(resourceDao.getRootProjectByComponentId(123L)).thenReturn(new ResourceDto().setId(123L)); + + byte[] dbContent = dryRunCache.getDatabaseForDryRun(123L); + assertThat(new String(dbContent)).isEqualTo("fake db content 1"); + + // Emulate invalidation of cache + Thread.sleep(100); + when(propertiesDao.selectProjectProperty(123L, DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY)).thenReturn(new PropertyDto().setValue("" + System.currentTimeMillis())); + + dbContent = dryRunCache.getDatabaseForDryRun(123L); + assertThat(new String(dbContent)).isEqualTo("fake db content 2"); + + verify(dryRunDatabaseFactory, times(2)).createNewDatabaseForDryRun(anyLong(), any(File.class), anyString()); } @Test - public void test_clean_all() throws Exception { + public void test_get_cache_location() throws Exception { File tempFolder = temp.newFolder(); when(serverFileSystem.getTempDir()).thenReturn(tempFolder); - File cacheLocation = dryRunCache.getCacheLocation(null); - FileUtils.forceMkdir(cacheLocation); - - dryRunCache.cleanAll(); - verify(propertiesDao).deleteAllProperties(DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY); - assertThat(cacheLocation).doesNotExist(); + assertThat(dryRunCache.getCacheLocation(null)).isEqualTo(new File(new File(tempFolder, "dryRun"), "default")); + assertThat(dryRunCache.getCacheLocation(123L)).isEqualTo(new File(new File(tempFolder, "dryRun"), "123")); } @Test - public void test_clean() throws Exception { + public void test_clean_all() throws Exception { File tempFolder = temp.newFolder(); when(serverFileSystem.getTempDir()).thenReturn(tempFolder); File cacheLocation = dryRunCache.getCacheLocation(null); FileUtils.forceMkdir(cacheLocation); - dryRunCache.clean(null); + dryRunCache.cleanAll(); + verify(propertiesDao).deleteAllProperties(DryRunCache.SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY); assertThat(cacheLocation).doesNotExist(); } diff --git a/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java b/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java index 323e8e68fba..a724df04af4 100644 --- a/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java +++ b/sonar-core/src/test/java/org/sonar/core/persistence/DryRunDatabaseFactoryTest.java @@ -19,7 +19,6 @@ */ package org.sonar.core.persistence; -import com.google.common.base.Charsets; import com.google.common.io.Files; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.io.FileUtils; @@ -28,17 +27,12 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.sonar.core.dryrun.DryRunCache; import java.io.File; import java.io.IOException; import java.sql.SQLException; import static org.fest.assertions.Assertions.assertThat; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { DryRunDatabaseFactory localDatabaseFactory; @@ -46,18 +40,10 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private File dryRunCacheFolder; - private DryRunCache dryRunCache; @Before public void setUp() throws Exception { - File tempFolder = temporaryFolder.newFolder(); - dryRunCacheFolder = new File(tempFolder, "dryRun"); - dryRunCache = mock(DryRunCache.class); - when(dryRunCache.getCacheLocation(anyLong())).thenReturn(dryRunCacheFolder); - when(dryRunCache.getModificationTimestamp(null)).thenReturn(0L); - when(dryRunCache.getModificationTimestamp(anyLong())).thenReturn(0L); - localDatabaseFactory = new DryRunDatabaseFactory(getDatabase(), dryRunCache); + localDatabaseFactory = new DryRunDatabaseFactory(getDatabase()); } @After @@ -71,59 +57,24 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { public void should_create_database_without_project() throws IOException, SQLException { setupData("should_create_database"); - assertThat(dryRunCacheFolder).doesNotExist(); - - byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); - dataSource = createDatabase(database); + byte[] db = createDb(null); + dataSource = createDatabase(db); assertThat(rowCount("metrics")).isEqualTo(2); assertThat(rowCount("projects")).isZero(); assertThat(rowCount("alerts")).isEqualTo(1); assertThat(rowCount("events")).isZero(); - - assertThat(dryRunCacheFolder).isDirectory(); } - @Test - public void should_reuse_database_without_project() throws IOException, SQLException { - setupData("should_create_database"); - - FileUtils.write(new File(dryRunCacheFolder, "123456.h2.db"), "fakeDbContent"); - - byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); - - assertThat(new String(database, Charsets.UTF_8)).isEqualTo("fakeDbContent"); - } - - @Test - public void should_evict_database_without_project() throws IOException, SQLException { - setupData("should_create_database"); - - // There is a DB in cache - File existingDb = new File(dryRunCacheFolder, "123456.h2.db"); - FileUtils.write(existingDb, "fakeDbContent"); - - // But last modification timestamp is greater - when(dryRunCache.getModificationTimestamp(null)).thenReturn(123457L); - - byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); - dataSource = createDatabase(database); - - assertThat(rowCount("metrics")).isEqualTo(2); - assertThat(rowCount("projects")).isZero(); - assertThat(rowCount("alerts")).isEqualTo(1); - - // Previous cached DB was deleted - verify(dryRunCache).clean(null); + private byte[] createDb(Long projectId) throws IOException { + return FileUtils.readFileToByteArray(localDatabaseFactory.createNewDatabaseForDryRun(projectId, temporaryFolder.newFolder(), "foo")); } @Test public void should_create_database_with_project() throws IOException, SQLException { setupData("should_create_database"); - assertThat(dryRunCacheFolder).doesNotExist(); - - byte[] database = localDatabaseFactory.createDatabaseForDryRun(123L); + byte[] database = createDb(123L); dataSource = createDatabase(database); assertThat(rowCount("metrics")).isEqualTo(2); @@ -131,49 +82,13 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { assertThat(rowCount("snapshots")).isEqualTo(1); assertThat(rowCount("project_measures")).isEqualTo(1); assertThat(rowCount("events")).isEqualTo(2); - - assertThat(dryRunCacheFolder).isDirectory(); - } - - @Test - public void should_reuse_database_with_project() throws IOException, SQLException { - setupData("should_create_database"); - - FileUtils.write(new File(dryRunCacheFolder, "123456.h2.db"), "fakeDbContent"); - - byte[] database = localDatabaseFactory.createDatabaseForDryRun(123L); - - assertThat(new String(database, Charsets.UTF_8)).isEqualTo("fakeDbContent"); - } - - @Test - public void should_evict_database_with_project() throws IOException, SQLException { - setupData("should_create_database"); - - // There is a DB in cache - File existingDb = new File(dryRunCacheFolder, "123456.h2.db"); - FileUtils.write(existingDb, "fakeDbContent"); - - // But last project modification timestamp is greater - when(dryRunCache.getModificationTimestamp(123L)).thenReturn(123457L); - - byte[] database = localDatabaseFactory.createDatabaseForDryRun(123L); - dataSource = createDatabase(database); - - assertThat(rowCount("metrics")).isEqualTo(2); - assertThat(rowCount("projects")).isEqualTo(1); - assertThat(rowCount("snapshots")).isEqualTo(1); - assertThat(rowCount("project_measures")).isEqualTo(1); - - // Previous cached DB was deleted - verify(dryRunCache).clean(123L); } @Test public void should_create_database_with_issues() throws IOException, SQLException { setupData("should_create_database_with_issues"); - byte[] database = localDatabaseFactory.createDatabaseForDryRun(399L); + byte[] database = createDb(399L); dataSource = createDatabase(database); assertThat(rowCount("issues")).isEqualTo(1); @@ -184,7 +99,7 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { setupData("multi-modules-with-issues"); // 300 : root module -> export issues of all modules - byte[] database = localDatabaseFactory.createDatabaseForDryRun(300L); + byte[] database = createDb(300L); dataSource = createDatabase(database); assertThat(rowCount("issues")).isEqualTo(1); assertThat(rowCount("projects")).isEqualTo(4); @@ -197,7 +112,7 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { setupData("multi-modules-with-issues"); // 301 : sub module with 1 closed issue and 1 open issue - byte[] database = localDatabaseFactory.createDatabaseForDryRun(301L); + byte[] database = createDb(301L); dataSource = createDatabase(database); assertThat(rowCount("issues")).isEqualTo(1); assertThat(rowCount("projects")).isEqualTo(2); @@ -210,7 +125,7 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { setupData("multi-modules-with-issues"); // 302 : sub module without any issues - byte[] database = localDatabaseFactory.createDatabaseForDryRun(302L); + byte[] database = createDb(302L); dataSource = createDatabase(database); assertThat(rowCount("issues")).isEqualTo(0); } @@ -219,7 +134,7 @@ public class DryRunDatabaseFactoryTest extends AbstractDaoTestCase { public void should_copy_permission_templates_data() throws Exception { setupData("should_copy_permission_templates"); - byte[] database = localDatabaseFactory.createDatabaseForDryRun(null); + byte[] database = createDb(null); dataSource = createDatabase(database); assertThat(rowCount("permission_templates")).isEqualTo(1); assertThat(rowCount("perm_templates_users")).isEqualTo(1); diff --git a/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java b/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java index acf58e4832a..340eb9ef3a1 100644 --- a/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java +++ b/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java @@ -46,11 +46,11 @@ import org.sonar.api.web.Page; import org.sonar.api.web.RubyRailsWebservice; import org.sonar.api.web.Widget; import org.sonar.core.component.SnapshotPerspectives; +import org.sonar.core.dryrun.DryRunCache; import org.sonar.core.i18n.RuleI18nManager; import org.sonar.core.measure.MeasureFilterEngine; import org.sonar.core.measure.MeasureFilterResult; import org.sonar.core.persistence.Database; -import org.sonar.core.persistence.DryRunDatabaseFactory; import org.sonar.core.purge.PurgeDao; import org.sonar.core.resource.ResourceIndexerDao; import org.sonar.core.resource.ResourceKeyUpdaterDao; @@ -497,7 +497,7 @@ public final class JRubyFacade { } public byte[] createDatabaseForDryRun(@Nullable Long projectId) { - return get(DryRunDatabaseFactory.class).createDatabaseForDryRun(projectId); + return get(DryRunCache.class).getDatabaseForDryRun(projectId); } public String getPeriodLabel(int periodIndex) { diff --git a/sonar-server/src/main/webapp/WEB-INF/app/controllers/alerts_controller.rb b/sonar-server/src/main/webapp/WEB-INF/app/controllers/alerts_controller.rb index 7eadc5417a9..e0dd6611b65 100644 --- a/sonar-server/src/main/webapp/WEB-INF/app/controllers/alerts_controller.rb +++ b/sonar-server/src/main/webapp/WEB-INF/app/controllers/alerts_controller.rb @@ -140,7 +140,7 @@ class AlertsController < ApplicationController private def reportGlobalModification - Property.set(Java::OrgSonarCoreDryrun::DryRunCache::SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY, java.lang.System.nanoTime) + Property.set(Java::OrgSonarCoreDryrun::DryRunCache::SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY, java.lang.System.currentTimeMillis) end end diff --git a/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb b/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb index 46d827121e5..36715cd1bed 100644 --- a/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb +++ b/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb @@ -35,6 +35,22 @@ class BatchBootstrapController < Api::ApiController send_data String.from_java_bytes(db_content) end + # PUT /batch_bootstrap/evict?project= + def evict + has_scan_role = has_role?(Java::OrgSonarCorePermission::Permission::SCAN_EXECUTION) + return render_unauthorized("You're not authorized to execute any SonarQube analysis. Please contact your SonarQube administrator.") if !has_scan_role + + project = load_project() + return render_unauthorized("You're not authorized to access to project '" + project.name + "', please contact your SonarQube administrator") if project && !has_scan_role && !has_role?(:user, project) + + if project + Property.set(Java::OrgSonarCoreDryrun::DryRunCache::SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY, java.lang.System.currentTimeMillis, project.root_project.id) + render_success('dryRun DB evicted') + else + render_bad_request('missing projectId') + end + end + # GET /batch_bootstrap/properties?[project=][&dryRun=true|false] def properties dryRun = params[:dryRun].present? && params[:dryRun] == "true" diff --git a/sonar-server/src/main/webapp/WEB-INF/app/controllers/project_controller.rb b/sonar-server/src/main/webapp/WEB-INF/app/controllers/project_controller.rb index 542a01636dd..cc4d9901857 100644 --- a/sonar-server/src/main/webapp/WEB-INF/app/controllers/project_controller.rb +++ b/sonar-server/src/main/webapp/WEB-INF/app/controllers/project_controller.rb @@ -400,7 +400,7 @@ class ProjectController < ApplicationController end def reportProjectModification(project_id) - Property.set(Java::OrgSonarCoreDryrun::DryRunCache::SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY, java.lang.System.nanoTime, project_id) + Property.set(Java::OrgSonarCoreDryrun::DryRunCache::SONAR_DRY_RUN_CACHE_LAST_UPDATE_KEY, java.lang.System.currentTimeMillis, project_id) end end -- 2.39.5