From 15efef19303c240d12ec345e774987ba8a075d8d Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Mon, 5 Mar 2012 14:05:37 +0100 Subject: Improve purge tasks of DbCleaner * improve logs * decrease the nb of SQL requests required to purge aborted builds and historical data of directories/files --- .../main/java/org/sonar/core/purge/PurgeDao.java | 72 +++++++++++------- .../java/org/sonar/core/resource/ResourceDao.java | 35 ++++++--- .../java/org/sonar/core/resource/ResourceDto.java | 6 +- .../org/sonar/core/resource/ResourceIndexDto.java | 12 +-- .../sonar/core/resource/ResourceIndexerMapper.java | 4 +- .../org/sonar/core/resource/ResourceMapper.java | 5 +- .../sonar/core/resource/ResourceIndexerMapper.xml | 4 +- .../org/sonar/core/resource/ResourceMapper.xml | 18 ++++- .../java/org/sonar/core/purge/PurgeDaoTest.java | 6 +- .../org/sonar/core/resource/ResourceDaoTest.java | 16 ++-- ...eHistoricalDataOfDirectoriesAndFiles-result.xml | 88 ++++++++++++++++++++++ ...ldDeleteHistoricalDataOfDirectoriesAndFiles.xml | 82 ++++++++++++++++++++ .../shouldPurgeDirectoriesAndFiles-result.xml | 88 ---------------------- .../shouldPurgeDirectoriesAndFiles.xml | 82 -------------------- 14 files changed, 279 insertions(+), 239 deletions(-) create mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml delete mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles-result.xml delete mode 100644 sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles.xml (limited to 'sonar-core') diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java index b448eeb8775..72322a2e180 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java @@ -25,8 +25,11 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.ibatis.session.ResultContext; import org.apache.ibatis.session.ResultHandler; import org.apache.ibatis.session.SqlSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.core.persistence.MyBatis; import org.sonar.core.resource.ResourceDao; +import org.sonar.core.resource.ResourceDto; import java.util.Collections; import java.util.List; @@ -34,6 +37,7 @@ import java.util.List; public class PurgeDao { private final MyBatis mybatis; private final ResourceDao resourceDao; + private static final Logger LOG = LoggerFactory.getLogger(PurgeDao.class); public PurgeDao(MyBatis mybatis, ResourceDao resourceDao) { this.mybatis = mybatis; @@ -44,15 +48,14 @@ public class PurgeDao { SqlSession session = mybatis.openBatchSession(); PurgeMapper purgeMapper = session.getMapper(PurgeMapper.class); try { - List projectIds = getProjectIds(rootResourceId, session); - for (Long projectId : projectIds) { - deleteAbortedBuilds(projectId, session, purgeMapper); - deleteHistoricalData(projectId, scopesWithoutHistoricalData, session, purgeMapper); - purge(projectId, session, purgeMapper); + List projects = getProjects(rootResourceId, session); + for (ResourceDto project : projects) { + deleteAbortedBuilds(project, session, purgeMapper); + purge(project, scopesWithoutHistoricalData, session, purgeMapper); } - for (Long projectId : projectIds) { - disableOrphanResources(projectId, session, purgeMapper); + for (ResourceDto project : projects) { + disableOrphanResources(project, session, purgeMapper); } } finally { MyBatis.closeQuietly(session); @@ -60,27 +63,39 @@ public class PurgeDao { return this; } - private void deleteAbortedBuilds(long projectId, SqlSession session, PurgeMapper purgeMapper) { - PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setStatus(new String[]{"U"}) - .setRootProjectId(projectId); - deleteSnapshots(query, session, purgeMapper); - } - - private void deleteHistoricalData(long projectId, String[] scopesWithoutHistoricalData, SqlSession session, PurgeMapper purgeMapper) { - if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) { + private void deleteAbortedBuilds(ResourceDto project, SqlSession session, PurgeMapper purgeMapper) { + if (hasAbortedBuilds(project.getId(), purgeMapper)) { + LOG.info("<- Deleting aborted builds of " + project.getLongName()); PurgeSnapshotQuery query = PurgeSnapshotQuery.create() - .setIslast(false) - .setScopes(scopesWithoutHistoricalData) - .setRootProjectId(projectId); + .setIslast(false) + .setStatus(new String[]{"U"}) + .setRootProjectId(project.getId()); deleteSnapshots(query, session, purgeMapper); } } - private void purge(final long projectId, final SqlSession session, final PurgeMapper purgeMapper) { - List projectSnapshotIds = purgeMapper.selectSnapshotIds(PurgeSnapshotQuery.create().setResourceId(projectId).setIslast(false).setNotPurged(true)); + private boolean hasAbortedBuilds(Long projectId, PurgeMapper purgeMapper) { + PurgeSnapshotQuery query = PurgeSnapshotQuery.create() + .setIslast(false) + .setStatus(new String[]{"U"}) + .setResourceId(projectId); + return !purgeMapper.selectSnapshotIds(query).isEmpty(); + } + + private void purge(final ResourceDto project, final String[] scopesWithoutHistoricalData, final SqlSession session, final PurgeMapper purgeMapper) { + List projectSnapshotIds = purgeMapper.selectSnapshotIds( + PurgeSnapshotQuery.create().setResourceId(project.getId()).setIslast(false).setNotPurged(true) + ); for (final Long projectSnapshotId : projectSnapshotIds) { + // TODO log date + if (!ArrayUtils.isEmpty(scopesWithoutHistoricalData)) { + PurgeSnapshotQuery query = PurgeSnapshotQuery.create() + .setIslast(false) + .setScopes(scopesWithoutHistoricalData) + .setRootSnapshotId(projectSnapshotId); + deleteSnapshots(query, session, purgeMapper); + } + PurgeSnapshotQuery query = PurgeSnapshotQuery.create().setRootSnapshotId(projectSnapshotId).setNotPurged(true); session.select("org.sonar.core.purge.PurgeMapper.selectSnapshotIds", query, new ResultHandler() { public void handleResult(ResultContext resultContext) { @@ -97,8 +112,8 @@ public class PurgeDao { session.commit(); } - private void disableOrphanResources(final long projectId, final SqlSession session, final PurgeMapper purgeMapper) { - session.select("org.sonar.core.purge.PurgeMapper.selectResourceIdsToDisable", projectId, new ResultHandler() { + private void disableOrphanResources(final ResourceDto project, final SqlSession session, final PurgeMapper purgeMapper) { + session.select("org.sonar.core.purge.PurgeMapper.selectResourceIdsToDisable", project.getId(), new ResultHandler() { public void handleResult(ResultContext resultContext) { Long resourceId = (Long) resultContext.getResultObject(); if (resourceId != null) { @@ -211,10 +226,11 @@ public class PurgeDao { /** * Load the whole tree of projects, including the project given in parameter. */ - private List getProjectIds(long rootProjectId, SqlSession session) { - List projectIds = Lists.newArrayList(rootProjectId); - projectIds.addAll(resourceDao.getDescendantProjectIds(rootProjectId, session)); - return projectIds; + private List getProjects(long rootProjectId, SqlSession session) { + List projects = Lists.newArrayList(); + projects.add(resourceDao.getResource(rootProjectId, session)); + projects.addAll(resourceDao.getDescendantProjects(rootProjectId, session)); + return projects; } @VisibleForTesting diff --git a/sonar-core/src/main/java/org/sonar/core/resource/ResourceDao.java b/sonar-core/src/main/java/org/sonar/core/resource/ResourceDao.java index 421eca7304d..e97fb1d7795 100644 --- a/sonar-core/src/main/java/org/sonar/core/resource/ResourceDao.java +++ b/sonar-core/src/main/java/org/sonar/core/resource/ResourceDao.java @@ -32,27 +32,40 @@ public class ResourceDao { this.mybatis = mybatis; } - public List getDescendantProjectIds(long projectId) { + public ResourceDto getResource(long projectId) { SqlSession session = mybatis.openSession(); try { - return getDescendantProjectIds(projectId, session); + return getResource(projectId, session); } finally { MyBatis.closeQuietly(session); } } - public List getDescendantProjectIds(long projectId, SqlSession session) { + public ResourceDto getResource(long projectId, SqlSession session) { + return session.getMapper(ResourceMapper.class).selectResource(projectId); + } + + public List getDescendantProjects(long projectId) { + SqlSession session = mybatis.openSession(); + try { + return getDescendantProjects(projectId, session); + } finally { + MyBatis.closeQuietly(session); + } + } + + public List getDescendantProjects(long projectId, SqlSession session) { ResourceMapper mapper = session.getMapper(ResourceMapper.class); - List ids = Lists.newArrayList(); - appendChildProjectIds(projectId, mapper, ids); - return ids; + List resources = Lists.newArrayList(); + appendChildProjects(projectId, mapper, resources); + return resources; } - private void appendChildProjectIds(long projectId, ResourceMapper mapper, List ids) { - List subProjectIds = mapper.selectDescendantProjectIds(projectId); - for (Long subProjectId : subProjectIds) { - ids.add(subProjectId); - appendChildProjectIds(subProjectId, mapper, ids); + private void appendChildProjects(long projectId, ResourceMapper mapper, List resources) { + List subProjects = mapper.selectDescendantProjects(projectId); + for (ResourceDto subProject : subProjects) { + resources.add(subProject); + appendChildProjects(subProject.getId(), mapper, resources); } } } diff --git a/sonar-core/src/main/java/org/sonar/core/resource/ResourceDto.java b/sonar-core/src/main/java/org/sonar/core/resource/ResourceDto.java index 8d0c3d2080c..4a672113359 100644 --- a/sonar-core/src/main/java/org/sonar/core/resource/ResourceDto.java +++ b/sonar-core/src/main/java/org/sonar/core/resource/ResourceDto.java @@ -21,18 +21,18 @@ package org.sonar.core.resource; public final class ResourceDto { - private Integer id; + private Long id; private String name; private String longName; private Integer rootId; private String scope; private String qualifier; - public Integer getId() { + public Long getId() { return id; } - public ResourceDto setId(Integer id) { + public ResourceDto setId(Long id) { this.id = id; return this; } diff --git a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexDto.java b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexDto.java index 2ef069dd27c..54bb5485953 100644 --- a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexDto.java +++ b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexDto.java @@ -24,8 +24,8 @@ public final class ResourceIndexDto { private String key; private int position; private int nameSize; - private int resourceId; - private int rootProjectId; + private long resourceId; + private long rootProjectId; private String qualifier; public String getKey() { @@ -46,20 +46,20 @@ public final class ResourceIndexDto { return this; } - public int getResourceId() { + public long getResourceId() { return resourceId; } - public ResourceIndexDto setResourceId(int i) { + public ResourceIndexDto setResourceId(long i) { this.resourceId = i; return this; } - public int getRootProjectId() { + public long getRootProjectId() { return rootProjectId; } - public ResourceIndexDto setRootProjectId(int i) { + public ResourceIndexDto setRootProjectId(long i) { this.rootProjectId = i; return this; } diff --git a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java index d6785a7ad3b..53f00b440e5 100644 --- a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java @@ -21,9 +21,9 @@ package org.sonar.core.resource; public interface ResourceIndexerMapper { - ResourceIndexDto selectMasterIndexByResourceId(int resourceId); + ResourceIndexDto selectMasterIndexByResourceId(long resourceId); - void deleteByResourceId(int resourceId); + void deleteByResourceId(long resourceId); void insert(ResourceIndexDto dto); } diff --git a/sonar-core/src/main/java/org/sonar/core/resource/ResourceMapper.java b/sonar-core/src/main/java/org/sonar/core/resource/ResourceMapper.java index 24bdb00a289..626dfd61f12 100644 --- a/sonar-core/src/main/java/org/sonar/core/resource/ResourceMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/resource/ResourceMapper.java @@ -22,6 +22,7 @@ package org.sonar.core.resource; import java.util.List; public interface ResourceMapper { - SnapshotDto selectSnapshotById(Long snapshotId); - List selectDescendantProjectIds(long rootProjectId); + SnapshotDto selectSnapshot(Long snapshotId); + ResourceDto selectResource(long id); + List selectDescendantProjects(long rootProjectId); } diff --git a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml index e5206a164be..b44c90dcc3e 100644 --- a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml @@ -41,13 +41,13 @@ and qualifier in ('TRK', 'VW', 'SVW') - select kee as "key", resource_id as "resourceId" from resource_index where resource_id=#{id} and position=0 - + delete from resource_index where resource_id=#{id} diff --git a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceMapper.xml b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceMapper.xml index f16cc1849dd..2245bab7513 100644 --- a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceMapper.xml @@ -22,13 +22,25 @@ + + + + + + + + + + - select * from snapshots where id=#{id} - + select * from projects where scope='PRJ' and root_id=#{id} diff --git a/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java b/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java index fbfe4661af2..0c6ea4e6d9a 100644 --- a/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java +++ b/sonar-core/src/test/java/org/sonar/core/purge/PurgeDaoTest.java @@ -125,10 +125,10 @@ public class PurgeDaoTest extends DaoTestCase { } @Test - public void shouldPurgeDirectoriesAndFiles() { - setupData("shouldPurgeDirectoriesAndFiles"); + public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { + setupData("shouldDeleteHistoricalDataOfDirectoriesAndFiles"); dao.purge(1, new String[]{Scopes.DIRECTORY, Scopes.FILE}); - checkTables("shouldPurgeDirectoriesAndFiles", "projects", "snapshots"); + checkTables("shouldDeleteHistoricalDataOfDirectoriesAndFiles", "projects", "snapshots"); } @Test diff --git a/sonar-core/src/test/java/org/sonar/core/resource/ResourceDaoTest.java b/sonar-core/src/test/java/org/sonar/core/resource/ResourceDaoTest.java index 0e22ee43154..bdda1f21aeb 100644 --- a/sonar-core/src/test/java/org/sonar/core/resource/ResourceDaoTest.java +++ b/sonar-core/src/test/java/org/sonar/core/resource/ResourceDaoTest.java @@ -26,6 +26,7 @@ import org.sonar.core.persistence.DaoTestCase; import java.util.List; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNot.not; import static org.hamcrest.core.IsNull.nullValue; import static org.junit.Assert.assertThat; @@ -41,23 +42,20 @@ public class ResourceDaoTest extends DaoTestCase { } @Test - public void testDescendantProjectIdsAndSelf() { + public void testDescendantProjects_do_not_include_self() { setupData("fixture"); - List ids = dao.getDescendantProjectIds(1L); + List resources = dao.getDescendantProjects(1L); - assertThat(ids, hasItems(2L)); - assertThat(ids.size(), Is.is(1)); + assertThat(resources.size(), Is.is(1)); + assertThat(resources.get(0).getId(), is(2L)); } @Test - public void testDescendantProjectIdsAndSelf_id_not_found() { + public void testDescendantProjects_id_not_found() { setupData("fixture"); - List ids = dao.getDescendantProjectIds(33333L); - - assertThat(ids, not(nullValue())); - assertThat(ids.size(), Is.is(0)); + assertThat(dao.getDescendantProjects(33333L).size(), Is.is(0)); } } diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml new file mode 100644 index 00000000000..c0f246cad7f --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml new file mode 100644 index 00000000000..03cbbfc847b --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles-result.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles-result.xml deleted file mode 100644 index a50174fdd99..00000000000 --- a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles-result.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles.xml b/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles.xml deleted file mode 100644 index 955b522db3f..00000000000 --- a/sonar-core/src/test/resources/org/sonar/core/purge/PurgeDaoTest/shouldPurgeDirectoriesAndFiles.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file -- cgit v1.2.3