From c07207bb758ad948ac8a36c91d2de39616ba4855 Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 30 Oct 2014 17:40:57 +0100 Subject: [PATCH] SONAR-5530 Synchronize issues by project --- .../server/component/db/ComponentDao.java | 5 +- .../server/search/IndexSynchronizer.java | 45 +++++++--- .../server/component/db/ComponentDaoTest.java | 7 ++ .../search/IndexSynchronizerMediumTest.java | 86 ++++++++++++++++--- .../ComponentDaoTest/find_project_uuids.xml | 47 ++++++++++ .../core/component/db/ComponentMapper.java | 6 +- .../core/component/db/ComponentMapper.xml | 10 +++ 7 files changed, 184 insertions(+), 22 deletions(-) create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_project_uuids.xml diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java b/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java index 37e19057d60..66952cc3f46 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java @@ -30,7 +30,6 @@ import org.sonar.server.db.BaseDao; import org.sonar.server.exceptions.NotFoundException; import javax.annotation.CheckForNull; - import java.util.Collection; import java.util.Collections; import java.util.List; @@ -144,4 +143,8 @@ public class ComponentDao extends BaseDao // TODO shouldn't we need to also delete snapshots ? mapper(session).deleteByKey(key); } + + public List findProjectUuids(DbSession session) { + return mapper(session).findProjectUuids(); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index ea2b1ca6a0a..462c9215bdd 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -19,6 +19,7 @@ */ package org.sonar.server.search; +import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.core.persistence.DbSession; @@ -26,11 +27,15 @@ import org.sonar.server.activity.index.ActivityIndex; import org.sonar.server.db.Dao; import org.sonar.server.db.DbClient; import org.sonar.server.issue.index.IssueAuthorizationIndex; +import org.sonar.server.issue.index.IssueAuthorizationNormalizer; import org.sonar.server.issue.index.IssueIndex; +import org.sonar.server.issue.index.IssueNormalizer; import org.sonar.server.qualityprofile.index.ActiveRuleIndex; import org.sonar.server.rule.index.RuleIndex; import java.util.Date; +import java.util.List; +import java.util.Map; /** * @since 4.4 @@ -50,16 +55,22 @@ public class IndexSynchronizer { public void execute() { /* synchronize all activeRules until we have mng tables in INDEX */ DbSession session = db.openSession(true); - LOG.info("Starting DB to Index synchronization"); - long start = System.currentTimeMillis(); - synchronize(session, db.ruleDao(), index.get(RuleIndex.class)); - synchronize(session, db.issueDao(), index.get(IssueIndex.class)); - synchronize(session, db.issueAuthorizationDao(), index.get(IssueAuthorizationIndex.class)); - synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class)); - synchronize(session, db.activityDao(), index.get(ActivityIndex.class)); - session.commit(); - LOG.info("Synchronization done in {}ms...", System.currentTimeMillis() - start); - session.close(); + try { + LOG.info("Starting DB to Index synchronization"); + long start = System.currentTimeMillis(); + List projectUuids = db.componentDao().findProjectUuids(session); + synchronize(session, db.ruleDao(), index.get(RuleIndex.class)); + synchronizeByProject(session, db.issueDao(), index.get(IssueIndex.class), + IssueNormalizer.IssueField.PROJECT.field(), projectUuids); + synchronizeByProject(session, db.issueAuthorizationDao(), index.get(IssueAuthorizationIndex.class), + IssueAuthorizationNormalizer.IssueAuthorizationField.PROJECT.field(), projectUuids); + synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class)); + synchronize(session, db.activityDao(), index.get(ActivityIndex.class)); + session.commit(); + LOG.info("Synchronization done in {}ms...", System.currentTimeMillis() - start); + } finally { + session.close(); + } } void synchronize(DbSession session, Dao dao, Index index) { @@ -73,4 +84,18 @@ public class IndexSynchronizer { dao.synchronizeAfter(session, lastSynch); } } + + void synchronizeByProject(DbSession session, Dao dao, Index index, String projectField, List projectUuids) { + Long count = index.getIndexStat().getDocumentCount(); + if (count <= 0) { + LOG.info("Initial indexing of {} records", index.getIndexType()); + dao.synchronizeAfter(session); + } else { + LOG.info("Synchronizing {} records for updates", index.getIndexType()); + for (String projectUuid : projectUuids) { + Map params = ImmutableMap.of(projectField, projectUuid); + dao.synchronizeAfter(session, index.getLastSynchronization(params), params); + } + } + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java index f528bba8f5a..283139c5f01 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java @@ -409,6 +409,13 @@ public class ComponentDaoTest extends AbstractDaoTestCase { checkTable("delete", "projects"); } + @Test + public void find_project_uuids() { + setupData("find_project_uuids"); + + assertThat(dao.findProjectUuids(session)).containsExactly("ABCD"); + } + @Test(expected = PersistenceException.class) public void synchronize_after() { dao.synchronizeAfter(session, new Date(0L)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java index a25da68803a..2e9569dc456 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java @@ -24,10 +24,20 @@ import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.sonar.api.rule.RuleKey; +import org.sonar.api.security.DefaultGroups; +import org.sonar.api.web.UserRole; +import org.sonar.core.component.ComponentDto; +import org.sonar.core.permission.PermissionFacade; import org.sonar.core.persistence.BatchSession; import org.sonar.core.persistence.DbSession; +import org.sonar.core.rule.RuleDto; +import org.sonar.server.component.ComponentTesting; import org.sonar.server.db.DbClient; +import org.sonar.server.issue.IssueTesting; +import org.sonar.server.issue.index.IssueAuthorizationIndex; +import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.rule.RuleTesting; +import org.sonar.server.rule.db.RuleDao; import org.sonar.server.rule.index.RuleIndex; import org.sonar.server.tester.ServerTester; @@ -54,9 +64,7 @@ public class IndexSynchronizerMediumTest { @After public void tearDown() throws Exception { - if (dbSession != null) { - dbSession.close(); - } + dbSession.close(); } @Test @@ -76,12 +84,70 @@ public class IndexSynchronizerMediumTest { tester.clearIndexes(); assertThat(indexClient.get(RuleIndex.class).countAll()).isEqualTo(0); - DbSession syncSession = dbClient.openSession(true); - try { - synchronizer.synchronize(syncSession, dbClient.ruleDao(), indexClient.get(RuleIndex.class)); - assertThat(indexClient.get(RuleIndex.class).countAll()).isEqualTo(numberOfRules); - } finally { - syncSession.close(); - } + synchronizer.execute(); + assertThat(indexClient.get(RuleIndex.class).countAll()).isEqualTo(numberOfRules); + } + + @Test + public void synchronize_issues_from_empty_index() throws Exception { + ComponentDto project = ComponentTesting.newProjectDto(); + ComponentDto file = ComponentTesting.newFileDto(project); + dbClient.componentDao().insert(dbSession, project, file); + + RuleDto rule = RuleTesting.newXooX1(); + tester.get(RuleDao.class).insert(dbSession, rule); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file, project)); + + dbSession.commit(); + + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + tester.clearIndexes(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(0); + + synchronizer.execute(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + } + + @Test + public void synchronize_issues_from_not_empty_index() throws Exception { + RuleDto rule = RuleTesting.newXooX1(); + tester.get(RuleDao.class).insert(dbSession, rule); + + ComponentDto project1 = ComponentTesting.newProjectDto(); + ComponentDto file1 = ComponentTesting.newFileDto(project1); + dbClient.componentDao().insert(dbSession, project1, file1); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file1, project1)); + + ComponentDto project2 = ComponentTesting.newProjectDto(); + ComponentDto file2 = ComponentTesting.newFileDto(project2); + dbClient.componentDao().insert(dbSession, project2, file2); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file2, project2)); + + dbSession.commit(); + + // Remove second issue to simulate that this issue has not been synchronized + indexClient.get(IssueIndex.class).deleteByProjectUuid(project2.uuid()); + + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + synchronizer.execute(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(2); + } + + @Test + public void synchronize_issues_authorization() throws Exception { + ComponentDto project = ComponentTesting.newProjectDto(); + ComponentDto file = ComponentTesting.newFileDto(project); + dbClient.componentDao().insert(dbSession, project, file); + + RuleDto rule = RuleTesting.newXooX1(); + tester.get(RuleDao.class).insert(dbSession, rule); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file, project)); + + tester.get(PermissionFacade.class).insertGroupPermission(project.getId(), DefaultGroups.ANYONE, UserRole.USER, dbSession); + dbSession.commit(); + + assertThat(indexClient.get(IssueAuthorizationIndex.class).countAll()).isEqualTo(0); + synchronizer.execute(); + assertThat(indexClient.get(IssueAuthorizationIndex.class).countAll()).isEqualTo(1); } } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_project_uuids.xml b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_project_uuids.xml new file mode 100644 index 00000000000..c4665e06e75 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_project_uuids.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java b/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java index 2f3ed28c008..4f25ede6e8f 100644 --- a/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java @@ -23,7 +23,6 @@ import org.apache.ibatis.annotations.Param; import org.sonar.core.component.ComponentDto; import javax.annotation.CheckForNull; - import java.util.Collection; import java.util.List; @@ -73,6 +72,11 @@ public interface ComponentMapper { */ List findByUuids(@Param("uuids") Collection uuids); + /** + * Return all project (PRJ/TRK) uuids + */ + List findProjectUuids(); + long countById(long id); void insert(ComponentDto rule); diff --git a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml index d51c13b7c88..3d643d6ba73 100644 --- a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml @@ -132,6 +132,16 @@ + + (kee, deprecated_kee, uuid, project_uuid, module_uuid, module_uuid_path, name, long_name, qualifier, scope, language, root_id, path, enabled, created_at, authorization_updated_at) -- 2.39.5