From 0e9234234737081869c10678aa4468e172156b25 Mon Sep 17 00:00:00 2001 From: Teryk Bellahsene Date: Wed, 5 Nov 2014 09:37:51 +0100 Subject: SONAR-5628 Project settings taken into account during the data cleaning of project analysis --- .../computation/dbcleaner/ProjectPurgeTask.java | 77 +++++++++++++++++++ .../dbcleaner/period/DefaultPeriodCleaner.java | 7 +- .../org/sonar/core/properties/PropertiesDao.java | 9 +++ .../sonar/core/properties/PropertiesMapper.java | 4 +- .../org/sonar/core/properties/PropertiesMapper.xml | 40 +++++----- .../dbcleaner/ProjectPurgeTaskTest.java | 88 ++++++++++++++++++++++ .../sonar/core/properties/PropertiesDaoTest.java | 17 ++++- .../selectProjectPropertiesByResourceId.xml | 22 ++++++ 8 files changed, 238 insertions(+), 26 deletions(-) create mode 100644 sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTask.java create mode 100644 sonar-core/src/test/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTaskTest.java create mode 100644 sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/selectProjectPropertiesByResourceId.xml (limited to 'sonar-core/src') diff --git a/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTask.java b/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTask.java new file mode 100644 index 00000000000..a2e1fe842ec --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTask.java @@ -0,0 +1,77 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.core.computation.dbcleaner; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.CoreProperties; +import org.sonar.api.ServerComponent; +import org.sonar.api.config.Settings; +import org.sonar.api.utils.TimeUtils; +import org.sonar.core.computation.dbcleaner.period.DefaultPeriodCleaner; +import org.sonar.core.purge.PurgeConfiguration; +import org.sonar.core.purge.PurgeDao; +import org.sonar.core.purge.PurgeProfiler; + +public class ProjectPurgeTask implements ServerComponent { + private static final Logger LOG = LoggerFactory.getLogger(ProjectPurgeTask.class); + private final PurgeProfiler profiler; + private final PurgeDao purgeDao; + private final DefaultPeriodCleaner periodCleaner; + + public ProjectPurgeTask(PurgeDao purgeDao, DefaultPeriodCleaner periodCleaner, PurgeProfiler profiler) { + this.purgeDao = purgeDao; + this.periodCleaner = periodCleaner; + this.profiler = profiler; + } + + public ProjectPurgeTask purge(PurgeConfiguration configuration, Settings settings) { + long start = System.currentTimeMillis(); + profiler.reset(); + cleanHistoricalData(configuration.rootProjectId(), settings); + doPurge(configuration); + if (settings.getBoolean(CoreProperties.PROFILING_LOG_PROPERTY)) { + long duration = System.currentTimeMillis() - start; + LOG.info("\n -------- Profiling for purge: " + TimeUtils.formatDuration(duration) + " --------\n"); + profiler.dump(duration, LOG); + LOG.info("\n -------- End of profiling for purge --------\n"); + } + return this; + } + + private void cleanHistoricalData(long resourceId, Settings settings) { + try { + periodCleaner.clean(resourceId, settings); + } catch (Exception e) { + // purge errors must no fail the batch + LOG.error("Fail to clean historical data [id=" + resourceId + "]", e); + } + } + + private void doPurge(PurgeConfiguration configuration) { + try { + purgeDao.purge(configuration); + } catch (Exception e) { + // purge errors must no fail the report analysis + LOG.error("Fail to purge data [id=" + configuration.rootProjectId() + "]", e); + } + } +} diff --git a/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/period/DefaultPeriodCleaner.java b/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/period/DefaultPeriodCleaner.java index a7f8a482533..bbb6a222ba8 100644 --- a/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/period/DefaultPeriodCleaner.java +++ b/sonar-core/src/main/java/org/sonar/core/computation/dbcleaner/period/DefaultPeriodCleaner.java @@ -25,7 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.ServerExtension; import org.sonar.api.config.Settings; -import org.sonar.api.resources.Project; import org.sonar.api.task.TaskExtension; import org.sonar.api.utils.DateUtils; import org.sonar.core.purge.PurgeDao; @@ -45,11 +44,11 @@ public class DefaultPeriodCleaner implements TaskExtension, ServerExtension { this.settings = settings; } - public void purge(Project project, int projectSnapshotId) { - clean(project.getId()); + public void clean(long projectId) { + clean(projectId, settings); } - public void clean(long projectId) { + public void clean(long projectId, Settings settings) { doClean(projectId, new Filters(settings).all()); } diff --git a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java index e7a13b39f26..6865434dfcd 100644 --- a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java +++ b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java @@ -99,6 +99,15 @@ public class PropertiesDao implements BatchComponent, ServerComponent, DaoCompon return session.getMapper(PropertiesMapper.class).selectProjectProperties(resourceKey); } + public List selectProjectProperties(long resourceId) { + SqlSession session = mybatis.openSession(false); + try { + return session.getMapper(PropertiesMapper.class).selectProjectPropertiesByResourceId(resourceId); + } finally { + MyBatis.closeQuietly(session); + } + } + public List selectProjectProperties(String resourceKey) { SqlSession session = mybatis.openSession(false); try { diff --git a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java index 6cdf28160af..10f298b53c3 100644 --- a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java @@ -35,6 +35,8 @@ public interface PropertiesMapper { List selectProjectProperties(String resourceKey); + List selectProjectPropertiesByResourceId(Long resourceId); + List selectSetOfResourceProperties(@Param("rId") Long projectId, @Param("propKeys") List propertyKeys); PropertyDto selectByKey(PropertyDto key); @@ -57,5 +59,5 @@ public interface PropertiesMapper { void renamePropertyKey(@Param("oldKey") String oldKey, @Param("newKey") String newKey); - void updateProperties(@Param("key") String key, @Param("oldValue")String oldValue, @Param("newValue") String newValue); + void updateProperties(@Param("key") String key, @Param("oldValue") String oldValue, @Param("newValue") String newValue); } diff --git a/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml b/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml index 250b49a361b..dedc3d19362 100644 --- a/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml @@ -20,23 +20,29 @@ FROM properties P, users U WHERE P.user_id = U.id AND P.prop_key = #{propKey} AND P.text_value LIKE 'true' AND ( - P.resource_id is null + P.resource_id is null OR P.resource_id in (select id from projects where kee=#{componentKey}) ) - select p.id as id, p.prop_key as "key", p.text_value as value, p.resource_id as resourceId, p.user_id as userId from properties p where p.resource_id is null and p.user_id is null - select p.id as id, p.prop_key as "key", p.text_value as value, p.resource_id as resourceId, p.user_id as userId from properties p, projects r - where p.resource_id=r.id and p.user_id is null and r.kee=#{id} + where p.resource_id=r.id and p.user_id is null and r.kee=#{resourceKey} + + + - + update properties set text_value = #{value} where id = #{id} - + INSERT INTO properties (prop_key, resource_id, user_id, text_value) VALUES (#{key}, #{resourceId}, #{userId}, #{value}) - + delete from properties where prop_key=#{key} and resource_id=#{rId} and user_id is null - + DELETE FROM properties WHERE - prop_key=#{key} - AND text_value LIKE #{value} - AND resource_id IS NOT NULL - AND user_id IS NULL + prop_key=#{key} + AND text_value LIKE #{value} + AND resource_id IS NOT NULL + AND user_id IS NULL - + delete from properties where prop_key=#{id} and resource_id is null and user_id is null - + delete from properties where resource_id is null and user_id is null - + delete from properties where prop_key=#{id} - + update properties set prop_key=#{newKey} where prop_key=#{oldKey} - + update properties set text_value=#{newValue} where text_value LIKE #{oldValue} and prop_key=#{key} diff --git a/sonar-core/src/test/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTaskTest.java b/sonar-core/src/test/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTaskTest.java new file mode 100644 index 00000000000..5b96c00a150 --- /dev/null +++ b/sonar-core/src/test/java/org/sonar/core/computation/dbcleaner/ProjectPurgeTaskTest.java @@ -0,0 +1,88 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.core.computation.dbcleaner; + +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.sonar.api.CoreProperties; +import org.sonar.api.config.Settings; +import org.sonar.core.computation.dbcleaner.period.DefaultPeriodCleaner; +import org.sonar.core.purge.PurgeConfiguration; +import org.sonar.core.purge.PurgeDao; +import org.sonar.core.purge.PurgeProfiler; + +import static org.mockito.Mockito.*; + +public class ProjectPurgeTaskTest { + + private ProjectPurgeTask sut; + private PurgeDao dao; + private PurgeProfiler profiler; + private DefaultPeriodCleaner periodCleaner; + + @Before + public void before() throws Exception { + this.dao = mock(PurgeDao.class); + this.profiler = mock(PurgeProfiler.class); + this.periodCleaner = mock(DefaultPeriodCleaner.class); + + this.sut = new ProjectPurgeTask(dao, periodCleaner, profiler); + } + + @Test + public void no_profiling_when_property_is_false() throws Exception { + Settings settings = mock(Settings.class); + when(settings.getBoolean(CoreProperties.PROFILING_LOG_PROPERTY)).thenReturn(false); + + sut.purge(mock(PurgeConfiguration.class), settings); + + verify(profiler, never()).dump(anyLong(), any(Logger.class)); + } + + @Test + public void profiling_when_property_is_true() throws Exception { + Settings settings = mock(Settings.class); + when(settings.getBoolean(CoreProperties.PROFILING_LOG_PROPERTY)).thenReturn(true); + + sut.purge(mock(PurgeConfiguration.class), settings); + + verify(profiler, times(1)).dump(anyLong(), any(Logger.class)); + } + + @Test + public void if_dao_purge_fails_it_should_not_interrupt_program_execution() throws Exception { + when(dao.purge(any(PurgeConfiguration.class))).thenThrow(NullPointerException.class); + + sut.purge(mock(PurgeConfiguration.class), mock(Settings.class)); + + verify(dao, times(1)).purge(any(PurgeConfiguration.class)); + } + + @Test + public void if_profiler_cleaning_fails_it_should_not_interrupt_program_execution() throws Exception { + doThrow(NullPointerException.class).when(periodCleaner).clean(anyLong(), any(Settings.class)); + + sut.purge(mock(PurgeConfiguration.class), mock(Settings.class)); + + verify(periodCleaner, times(1)).clean(anyLong(), any(Settings.class)); + } +} diff --git a/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java b/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java index 807bfec1775..454ba2829c2 100644 --- a/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java +++ b/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java @@ -36,12 +36,10 @@ import static org.junit.Assert.assertThat; public class PropertiesDaoTest extends AbstractDaoTestCase { - private DbSession session; - - private PropertiesDao dao; - @Rule public ExpectedException thrown = ExpectedException.none(); + private DbSession session; + private PropertiesDao dao; @Before public void createDao() { @@ -143,6 +141,17 @@ public class PropertiesDaoTest extends AbstractDaoTestCase { assertThat(first.getValue(), is("one")); } + @Test + public void selectProjectPropertiesByResourceId() { + setupData("selectProjectPropertiesByResourceId"); + + List properties = dao.selectProjectProperties(10L); + + assertThat(properties.size(), is(2)); + assertThat(properties).onProperty("key").containsOnly("struts.one", "user.two"); + assertThat(properties).onProperty("value").containsOnly("one", "two"); + } + @Test public void selectProjectProperty() { setupData("selectProjectProperties"); diff --git a/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/selectProjectPropertiesByResourceId.xml b/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/selectProjectPropertiesByResourceId.xml new file mode 100644 index 00000000000..52fec852a8d --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/selectProjectPropertiesByResourceId.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + -- cgit v1.2.3