diff options
author | Julien Lancelot <julien.lancelot@gmail.com> | 2014-10-30 17:40:57 +0100 |
---|---|---|
committer | Julien Lancelot <julien.lancelot@gmail.com> | 2014-10-30 17:41:23 +0100 |
commit | c07207bb758ad948ac8a36c91d2de39616ba4855 (patch) | |
tree | 088fb1898e7b9bc7862300196268dc7cbce3dc60 /server | |
parent | 09c06b60675786f7d595e3800596519f92e010a7 (diff) | |
download | sonarqube-c07207bb758ad948ac8a36c91d2de39616ba4855.tar.gz sonarqube-c07207bb758ad948ac8a36c91d2de39616ba4855.zip |
SONAR-5530 Synchronize issues by project
Diffstat (limited to 'server')
5 files changed, 169 insertions, 21 deletions
diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java b/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java index 37e19057d60..66952cc3f46 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java @@ -30,7 +30,6 @@ import org.sonar.server.db.BaseDao; import org.sonar.server.exceptions.NotFoundException; import javax.annotation.CheckForNull; - import java.util.Collection; import java.util.Collections; import java.util.List; @@ -144,4 +143,8 @@ public class ComponentDao extends BaseDao<ComponentMapper, ComponentDto, String> // TODO shouldn't we need to also delete snapshots ? mapper(session).deleteByKey(key); } + + public List<String> findProjectUuids(DbSession session) { + return mapper(session).findProjectUuids(); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index ea2b1ca6a0a..462c9215bdd 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -19,6 +19,7 @@ */ package org.sonar.server.search; +import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.core.persistence.DbSession; @@ -26,11 +27,15 @@ import org.sonar.server.activity.index.ActivityIndex; import org.sonar.server.db.Dao; import org.sonar.server.db.DbClient; import org.sonar.server.issue.index.IssueAuthorizationIndex; +import org.sonar.server.issue.index.IssueAuthorizationNormalizer; import org.sonar.server.issue.index.IssueIndex; +import org.sonar.server.issue.index.IssueNormalizer; import org.sonar.server.qualityprofile.index.ActiveRuleIndex; import org.sonar.server.rule.index.RuleIndex; import java.util.Date; +import java.util.List; +import java.util.Map; /** * @since 4.4 @@ -50,16 +55,22 @@ public class IndexSynchronizer { public void execute() { /* synchronize all activeRules until we have mng tables in INDEX */ DbSession session = db.openSession(true); - LOG.info("Starting DB to Index synchronization"); - long start = System.currentTimeMillis(); - synchronize(session, db.ruleDao(), index.get(RuleIndex.class)); - synchronize(session, db.issueDao(), index.get(IssueIndex.class)); - synchronize(session, db.issueAuthorizationDao(), index.get(IssueAuthorizationIndex.class)); - synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class)); - synchronize(session, db.activityDao(), index.get(ActivityIndex.class)); - session.commit(); - LOG.info("Synchronization done in {}ms...", System.currentTimeMillis() - start); - session.close(); + try { + LOG.info("Starting DB to Index synchronization"); + long start = System.currentTimeMillis(); + List<String> projectUuids = db.componentDao().findProjectUuids(session); + synchronize(session, db.ruleDao(), index.get(RuleIndex.class)); + synchronizeByProject(session, db.issueDao(), index.get(IssueIndex.class), + IssueNormalizer.IssueField.PROJECT.field(), projectUuids); + synchronizeByProject(session, db.issueAuthorizationDao(), index.get(IssueAuthorizationIndex.class), + IssueAuthorizationNormalizer.IssueAuthorizationField.PROJECT.field(), projectUuids); + synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class)); + synchronize(session, db.activityDao(), index.get(ActivityIndex.class)); + session.commit(); + LOG.info("Synchronization done in {}ms...", System.currentTimeMillis() - start); + } finally { + session.close(); + } } void synchronize(DbSession session, Dao dao, Index index) { @@ -73,4 +84,18 @@ public class IndexSynchronizer { dao.synchronizeAfter(session, lastSynch); } } + + void synchronizeByProject(DbSession session, Dao dao, Index index, String projectField, List<String> projectUuids) { + Long count = index.getIndexStat().getDocumentCount(); + if (count <= 0) { + LOG.info("Initial indexing of {} records", index.getIndexType()); + dao.synchronizeAfter(session); + } else { + LOG.info("Synchronizing {} records for updates", index.getIndexType()); + for (String projectUuid : projectUuids) { + Map<String, String> params = ImmutableMap.of(projectField, projectUuid); + dao.synchronizeAfter(session, index.getLastSynchronization(params), params); + } + } + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java index f528bba8f5a..283139c5f01 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java @@ -409,6 +409,13 @@ public class ComponentDaoTest extends AbstractDaoTestCase { checkTable("delete", "projects"); } + @Test + public void find_project_uuids() { + setupData("find_project_uuids"); + + assertThat(dao.findProjectUuids(session)).containsExactly("ABCD"); + } + @Test(expected = PersistenceException.class) public void synchronize_after() { dao.synchronizeAfter(session, new Date(0L)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java index a25da68803a..2e9569dc456 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java @@ -24,10 +24,20 @@ import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.sonar.api.rule.RuleKey; +import org.sonar.api.security.DefaultGroups; +import org.sonar.api.web.UserRole; +import org.sonar.core.component.ComponentDto; +import org.sonar.core.permission.PermissionFacade; import org.sonar.core.persistence.BatchSession; import org.sonar.core.persistence.DbSession; +import org.sonar.core.rule.RuleDto; +import org.sonar.server.component.ComponentTesting; import org.sonar.server.db.DbClient; +import org.sonar.server.issue.IssueTesting; +import org.sonar.server.issue.index.IssueAuthorizationIndex; +import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.rule.RuleTesting; +import org.sonar.server.rule.db.RuleDao; import org.sonar.server.rule.index.RuleIndex; import org.sonar.server.tester.ServerTester; @@ -54,9 +64,7 @@ public class IndexSynchronizerMediumTest { @After public void tearDown() throws Exception { - if (dbSession != null) { - dbSession.close(); - } + dbSession.close(); } @Test @@ -76,12 +84,70 @@ public class IndexSynchronizerMediumTest { tester.clearIndexes(); assertThat(indexClient.get(RuleIndex.class).countAll()).isEqualTo(0); - DbSession syncSession = dbClient.openSession(true); - try { - synchronizer.synchronize(syncSession, dbClient.ruleDao(), indexClient.get(RuleIndex.class)); - assertThat(indexClient.get(RuleIndex.class).countAll()).isEqualTo(numberOfRules); - } finally { - syncSession.close(); - } + synchronizer.execute(); + assertThat(indexClient.get(RuleIndex.class).countAll()).isEqualTo(numberOfRules); + } + + @Test + public void synchronize_issues_from_empty_index() throws Exception { + ComponentDto project = ComponentTesting.newProjectDto(); + ComponentDto file = ComponentTesting.newFileDto(project); + dbClient.componentDao().insert(dbSession, project, file); + + RuleDto rule = RuleTesting.newXooX1(); + tester.get(RuleDao.class).insert(dbSession, rule); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file, project)); + + dbSession.commit(); + + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + tester.clearIndexes(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(0); + + synchronizer.execute(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + } + + @Test + public void synchronize_issues_from_not_empty_index() throws Exception { + RuleDto rule = RuleTesting.newXooX1(); + tester.get(RuleDao.class).insert(dbSession, rule); + + ComponentDto project1 = ComponentTesting.newProjectDto(); + ComponentDto file1 = ComponentTesting.newFileDto(project1); + dbClient.componentDao().insert(dbSession, project1, file1); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file1, project1)); + + ComponentDto project2 = ComponentTesting.newProjectDto(); + ComponentDto file2 = ComponentTesting.newFileDto(project2); + dbClient.componentDao().insert(dbSession, project2, file2); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file2, project2)); + + dbSession.commit(); + + // Remove second issue to simulate that this issue has not been synchronized + indexClient.get(IssueIndex.class).deleteByProjectUuid(project2.uuid()); + + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + synchronizer.execute(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(2); + } + + @Test + public void synchronize_issues_authorization() throws Exception { + ComponentDto project = ComponentTesting.newProjectDto(); + ComponentDto file = ComponentTesting.newFileDto(project); + dbClient.componentDao().insert(dbSession, project, file); + + RuleDto rule = RuleTesting.newXooX1(); + tester.get(RuleDao.class).insert(dbSession, rule); + dbClient.issueDao().insert(dbSession, IssueTesting.newDto(rule, file, project)); + + tester.get(PermissionFacade.class).insertGroupPermission(project.getId(), DefaultGroups.ANYONE, UserRole.USER, dbSession); + dbSession.commit(); + + assertThat(indexClient.get(IssueAuthorizationIndex.class).countAll()).isEqualTo(0); + synchronizer.execute(); + assertThat(indexClient.get(IssueAuthorizationIndex.class).countAll()).isEqualTo(1); } } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_project_uuids.xml b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_project_uuids.xml new file mode 100644 index 00000000000..c4665e06e75 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_project_uuids.xml @@ -0,0 +1,47 @@ +<dataset> + + <!-- root project --> + <projects id="1" root_id="[null]" scope="PRJ" qualifier="TRK" kee="org.struts:struts" deprecated_kee="org.struts:struts" name="Struts" + uuid="ABCD" project_uuid="ABCD" module_uuid="[null]" module_uuid_path="[null]" + description="the description" long_name="Apache Struts" + enabled="[true]" language="[null]" copy_resource_id="[null]" person_id="[null]" path="[null]" authorization_updated_at="2014-06-18" /> + + <!-- module --> + <projects id="2" root_id="1" kee="org.struts:struts-core" name="Struts Core" + uuid="EFGH" project_uuid="ABCD" module_uuid="[null]" module_uuid_path="ABCD." + scope="PRJ" qualifier="BRC" long_name="Struts Core" + description="[null]" enabled="[true]" language="[null]" copy_resource_id="[null]" person_id="[null]" authorization_updated_at="[null]" /> + + <!-- directory --> + <projects long_name="org.struts" id="3" scope="DIR" qualifier="DIR" kee="org.struts:struts-core:src/org/struts" + uuid="GHIJ" project_uuid="ABCD" module_uuid="EFGH" module_uuid_path="ABCD.EFGH." + name="src/org/struts" root_id="2" + description="[null]" + enabled="[true]" language="[null]" copy_resource_id="[null]" person_id="[null]" path="src/org/struts" authorization_updated_at="[null]" /> + + <!-- file --> + <projects long_name="org.struts.RequestContext" id="4" scope="FIL" qualifier="FIL" kee="org.struts:struts-core:src/org/struts/RequestContext.java" + uuid="KLMN" project_uuid="ABCD" module_uuid="EFGH" module_uuid_path="ABCD.EFGH." + name="RequestContext.java" root_id="2" + description="[null]" + enabled="[true]" language="java" copy_resource_id="[null]" person_id="[null]" path="src/org/struts/RequestContext.java" authorization_updated_at="[null]" /> + + <!-- Disabled projects --> + <projects id="10" root_id="[null]" scope="PRJ" qualifier="TRK" kee="org.disabled.project" name="Disabled Project" + uuid="DCBA" project_uuid="DCBA" module_uuid="[null]" module_uuid_path="[null]" + description="the description" long_name="Disabled project" + enabled="[false]" language="[null]" copy_resource_id="[null]" person_id="[null]" path="[null]" authorization_updated_at="2014-06-18" /> + + <!-- Developer --> + <projects id="20" kee="DEV:developer@company.net" name="developer@company.net" long_name="Developer" scope="PRJ" qualifier="DEV" root_id="[null]" description="[null]" + uuid="DEV" project_uuid="[null]" module_uuid="[null]" module_uuid_path="[null]" + enabled="[true]" language="java" copy_resource_id="[null]" person_id="[null]" path="[null]" deprecated_kee="[null]" + created_at="2014-09-01" authorization_updated_at="[null]"/> + + <!-- View --> + <projects id="30" kee="view" name="View" long_name="View" scope="PRJ" qualifier="VW" root_id="[null]" description="[null]" + uuid="VIEW" project_uuid="[null]" module_uuid="[null]" module_uuid_path="[null]" + enabled="[true]" language="java" copy_resource_id="[null]" person_id="[null]" path="[null]" deprecated_kee="[null]" + created_at="2014-09-01" authorization_updated_at="[null]"/> + +</dataset> |