From f05fa4020e8f937c9f6819d0fdab31aada97a78e Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Wed, 12 Oct 2016 18:31:41 +0200 Subject: [PATCH] SONAR-8229 Update project measures index when removing a project --- .../component/ComponentCleanerService.java | 6 +- .../project/es/ProjectMeasuresIndexer.java | 8 + .../ComponentCleanerServiceTest.java | 282 ++++++++++++++++++ .../es/ProjectMeasuresIndexerTest.java | 39 ++- .../project/ws/BulkDeleteActionTest.java | 6 +- .../server/project/ws/DeleteActionTest.java | 2 + 6 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java index 4026a58147c..11153d2bbb4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java @@ -31,6 +31,7 @@ import org.sonar.db.MyBatis; import org.sonar.db.component.ComponentDto; import org.sonar.server.issue.index.IssueAuthorizationIndexer; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.project.es.ProjectMeasuresIndexer; import org.sonar.server.test.index.TestIndexer; @ServerSide @@ -41,15 +42,17 @@ public class ComponentCleanerService { private final IssueAuthorizationIndexer issueAuthorizationIndexer; private final IssueIndexer issueIndexer; private final TestIndexer testIndexer; + private final ProjectMeasuresIndexer projectMeasuresIndexer; private final ResourceTypes resourceTypes; private final ComponentFinder componentFinder; public ComponentCleanerService(DbClient dbClient, IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, - TestIndexer testIndexer, ResourceTypes resourceTypes, ComponentFinder componentFinder) { + TestIndexer testIndexer, ProjectMeasuresIndexer projectMeasuresIndexer, ResourceTypes resourceTypes, ComponentFinder componentFinder) { this.dbClient = dbClient; this.issueAuthorizationIndexer = issueAuthorizationIndexer; this.issueIndexer = issueIndexer; this.testIndexer = testIndexer; + this.projectMeasuresIndexer = projectMeasuresIndexer; this.resourceTypes = resourceTypes; this.componentFinder = componentFinder; } @@ -85,6 +88,7 @@ public class ComponentCleanerService { issueAuthorizationIndexer.deleteProject(projectUuid, false); issueIndexer.deleteProject(projectUuid); testIndexer.deleteByProject(projectUuid); + projectMeasuresIndexer.deleteProject(projectUuid); } private static boolean hasNotProjectScope(ComponentDto project) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java index 0b933b0950e..7efb8eab914 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java @@ -51,6 +51,14 @@ public class ProjectMeasuresIndexer extends BaseIndexer { index(lastUpdatedAt -> doIndex(createBulkIndexer(false), projectUuid)); } + public void deleteProject(String uuid) { + esClient + .prepareDelete(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES, uuid) + .setRefresh(true) + .setRouting(uuid) + .get(); + } + private long doIndex(BulkIndexer bulk, @Nullable String projectUuid) { DbSession dbSession = dbClient.openSession(false); try { diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceTest.java new file mode 100644 index 00000000000..cb512d5e502 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceTest.java @@ -0,0 +1,282 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.component; + +import java.util.Date; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.config.MapSettings; +import org.sonar.api.resources.ResourceType; +import org.sonar.api.resources.ResourceTypes; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.utils.System2; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.SnapshotDto; +import org.sonar.db.component.SnapshotTesting; +import org.sonar.db.issue.IssueDto; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleTesting; +import org.sonar.server.es.EsTester; +import org.sonar.server.exceptions.NotFoundException; +import org.sonar.server.issue.IssueTesting; +import org.sonar.server.issue.index.IssueAuthorizationDoc; +import org.sonar.server.issue.index.IssueAuthorizationIndexer; +import org.sonar.server.issue.index.IssueIndexDefinition; +import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.project.es.ProjectMeasuresIndexDefinition; +import org.sonar.server.project.es.ProjectMeasuresIndexer; +import org.sonar.server.test.index.TestDoc; +import org.sonar.server.test.index.TestIndexDefinition; +import org.sonar.server.test.index.TestIndexer; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.ComponentTesting.newFileDto; +import static org.sonar.db.component.ComponentTesting.newProjectDto; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_AUTHORIZATION; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; + +public class ComponentCleanerServiceTest { + + @Rule + public DbTester db = DbTester.create(System2.INSTANCE); + + @Rule + public EsTester es = new EsTester( + new IssueIndexDefinition(new MapSettings()), + new TestIndexDefinition(new MapSettings()), + new ProjectMeasuresIndexDefinition(new MapSettings())); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + DbClient dbClient = db.getDbClient(); + DbSession dbSession = db.getSession(); + + IssueAuthorizationIndexer issueAuthorizationIndexer = new IssueAuthorizationIndexer(dbClient, es.client()); + IssueIndexer issueIndexer = new IssueIndexer(dbClient, es.client()); + TestIndexer testIndexer = new TestIndexer(dbClient, es.client()); + ProjectMeasuresIndexer projectMeasuresIndexer = new ProjectMeasuresIndexer(dbClient, es.client()); + + ResourceTypes mockResourceTypes = mock(ResourceTypes.class); + + ComponentCleanerService underTest = new ComponentCleanerService(dbClient, + issueAuthorizationIndexer, issueIndexer, testIndexer, projectMeasuresIndexer, + mockResourceTypes, + new ComponentFinder(dbClient)); + + @Test + public void delete_project_by_key_in_db() { + DbData data1 = insertDataInDb(1); + DbData data2 = insertDataInDb(2); + + underTest.delete(data1.project.key()); + + assertDataDoesNotExistInDB(data1); + assertDataStillExistsInDb(data2); + } + + @Test + public void delete_project_by_key_in_index() throws Exception { + IndexData data1 = insertDataInEs(1); + IndexData data2 = insertDataInEs(2); + + underTest.delete(data1.project.key()); + + assertDataDoesNotExistInIndex(data1); + assertDataStillExistsInIndex(data2); + } + + @Test + public void delete_projects_in_db() { + DbData data1 = insertDataInDb(1); + DbData data2 = insertDataInDb(2); + DbData data3 = insertDataInDb(3); + + underTest.delete(dbSession, asList(data1.project, data2.project)); + dbSession.commit(); + + assertDataDoesNotExistInDB(data1); + assertDataDoesNotExistInDB(data2); + assertDataStillExistsInDb(data3); + } + + @Test + public void delete_projects_in_index() throws Exception { + IndexData data1 = insertDataInEs(1); + IndexData data2 = insertDataInEs(2); + IndexData data3 = insertDataInEs(3); + + underTest.delete(dbSession, asList(data1.project, data2.project)); + dbSession.commit(); + + assertDataDoesNotExistInIndex(data1); + assertDataDoesNotExistInIndex(data2); + assertDataStillExistsInIndex(data3); + } + + @Test + public void fail_to_delete_unknown_project() throws Exception { + expectedException.expect(NotFoundException.class); + underTest.delete("unknown"); + } + + @Test + public void fail_to_delete_not_project_scope() throws Exception { + mockResourceTypeAsValidProject(); + ComponentDto project = dbClient.componentDao().insert(dbSession, newProjectDto()); + ComponentDto file = dbClient.componentDao().insert(dbSession, newFileDto(project, null)); + dbSession.commit(); + + expectedException.expect(IllegalArgumentException.class); + underTest.delete(file.key()); + } + + @Test + public void fail_to_delete_not_deletable_resource_type() throws Exception { + ResourceType resourceType = mock(ResourceType.class); + when(resourceType.getBooleanProperty("deletable")).thenReturn(false); + when(mockResourceTypes.get(anyString())).thenReturn(resourceType); + ComponentDto project = dbClient.componentDao().insert(dbSession, newProjectDto()); + dbSession.commit(); + + expectedException.expect(IllegalArgumentException.class); + underTest.delete(project.key()); + } + + @Test + public void fail_to_delete_null_resource_type() throws Exception { + when(mockResourceTypes.get(anyString())).thenReturn(null); + ComponentDto project = dbClient.componentDao().insert(dbSession, newProjectDto()); + dbSession.commit(); + + expectedException.expect(IllegalArgumentException.class); + underTest.delete(project.key()); + } + + private DbData insertDataInDb(int id) { + String suffix = String.valueOf(id); + ComponentDto project = newProjectDto("project-uuid-" + suffix) + .setKey("project-key-" + suffix); + RuleDto rule = RuleTesting.newDto(RuleKey.of("sonarqube", "rule-" + suffix)); + dbClient.ruleDao().insert(dbSession, rule); + IssueDto issue = IssueTesting.newDto(rule, project, project).setKee("issue-key-" + suffix).setUpdatedAt(new Date().getTime()); + dbClient.componentDao().insert(dbSession, project); + SnapshotDto snapshot = dbClient.snapshotDao().insert(dbSession, SnapshotTesting.newAnalysis(project)); + dbClient.issueDao().insert(dbSession, issue); + dbSession.commit(); + mockResourceTypeAsValidProject(); + return new DbData(project, snapshot, issue); + } + + private void mockResourceTypeAsValidProject() { + ResourceType resourceType = mock(ResourceType.class); + when(resourceType.getBooleanProperty(anyString())).thenReturn(true); + when(mockResourceTypes.get(anyString())).thenReturn(resourceType); + } + + private void assertDataStillExistsInDb(DbData data) { + assertDataInDb(data, true); + } + + private void assertDataDoesNotExistInDB(DbData data) { + assertDataInDb(data, false); + } + + private void assertDataInDb(DbData data, boolean exists) { + assertThat(dbClient.componentDao().selectByUuid(dbSession, data.project.uuid()).isPresent()).isEqualTo(exists); + assertThat(dbClient.snapshotDao().selectByUuid(dbSession, data.snapshot.getUuid()).isPresent()).isEqualTo(exists); + assertThat(dbClient.issueDao().selectByKey(dbSession, data.issue.getKey()).isPresent()).isEqualTo(exists); + } + + private IndexData insertDataInEs(int id) throws Exception { + mockResourceTypeAsValidProject(); + + String suffix = String.valueOf(id); + ComponentDto project = newProjectDto("project-uuid-" + suffix) + .setKey("project-key-" + suffix); + dbClient.componentDao().insert(dbSession, project); + dbSession.commit(); + projectMeasuresIndexer.index(); + + String issueKey = "issue-key-" + suffix; + es.putDocuments(IssueIndexDefinition.INDEX, TYPE_ISSUE, IssueTesting.newDoc(issueKey, project)); + es.putDocuments(IssueIndexDefinition.INDEX, TYPE_AUTHORIZATION, new IssueAuthorizationDoc().setProjectUuid(project.uuid())); + + TestDoc testDoc = new TestDoc().setUuid("test-uuid-" + suffix).setProjectUuid(project.uuid()).setFileUuid(project.uuid()); + es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, testDoc); + + return new IndexData(project, issueKey, testDoc.getId()); + } + + private void assertDataStillExistsInIndex(IndexData data) { + assertDataInIndex(data, true); + } + + private void assertDataDoesNotExistInIndex(IndexData data) { + assertDataInIndex(data, false); + } + + private void assertDataInIndex(IndexData data, boolean exists) { + if (exists) { + assertThat(es.getIds(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE)).contains(data.issueKey); + assertThat(es.getIds(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION)).contains(data.project.uuid()); + assertThat(es.getIds(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE)).contains(data.testId); + assertThat(es.getIds(ProjectMeasuresIndexDefinition.INDEX_PROJECT_MEASURES, ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).contains(data.project.uuid()); + } else { + assertThat(es.getIds(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE)).doesNotContain(data.issueKey); + assertThat(es.getIds(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION)).doesNotContain(data.project.uuid()); + assertThat(es.getIds(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE)).doesNotContain(data.testId); + assertThat(es.getIds(ProjectMeasuresIndexDefinition.INDEX_PROJECT_MEASURES, ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).doesNotContain(data.project.uuid()); + } + } + + private static class DbData { + final ComponentDto project; + final SnapshotDto snapshot; + final IssueDto issue; + + public DbData(ComponentDto project, SnapshotDto snapshot, IssueDto issue) { + this.project = project; + this.snapshot = snapshot; + this.issue = issue; + } + } + + private static class IndexData { + final ComponentDto project; + final String issueKey; + final String testId; + + public IndexData(ComponentDto project, String issueKey, String testId) { + this.project = project; + this.issueKey = issueKey; + this.testId = testId; + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresIndexerTest.java index 6b562b439bc..3d91b1b9a3b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresIndexerTest.java @@ -62,10 +62,21 @@ public class ProjectMeasuresIndexerTest { @Test public void index_all_project() { componentDbTester.insertProjectAndSnapshot(newProjectDto()); + componentDbTester.insertProjectAndSnapshot(newProjectDto()); + componentDbTester.insertProjectAndSnapshot(newProjectDto()); + + underTest.index(); + + assertThat(esTester.countDocuments(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).isEqualTo(3); + } + + @Test + public void index_projects_even_when_no_analysis() { + ComponentDto project = componentDbTester.insertProject(); underTest.index(); - assertThat(esTester.countDocuments(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).isEqualTo(1); + assertThat(esTester.getIds(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).containsOnly(project.uuid()); } @Test @@ -104,4 +115,30 @@ public class ProjectMeasuresIndexerTest { .must(termQuery(ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT, new Date(analysis.getCreatedAt()))))); assertThat(request.get().getHits()).hasSize(1); } + + @Test + public void delete_project() { + ComponentDto project1 = newProjectDto(); + componentDbTester.insertProjectAndSnapshot(project1); + ComponentDto project2 = newProjectDto(); + componentDbTester.insertProjectAndSnapshot(project2); + ComponentDto project3 = newProjectDto(); + componentDbTester.insertProjectAndSnapshot(project3); + underTest.index(); + + underTest.deleteProject(project1.uuid()); + + assertThat(esTester.getIds(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).containsOnly(project2.uuid(), project3.uuid()); + } + + @Test + public void does_nothing_when_deleting_unknown_project() throws Exception { + ComponentDto project = newProjectDto(); + componentDbTester.insertProjectAndSnapshot(project); + underTest.index(); + + underTest.deleteProject("UNKNOWN"); + + assertThat(esTester.getIds(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).containsOnly(project.uuid()); + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java index a4a7c29cc92..4d7a3da3ea8 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java @@ -51,6 +51,7 @@ import org.sonar.server.issue.index.IssueAuthorizationDoc; import org.sonar.server.issue.index.IssueAuthorizationIndexer; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.project.es.ProjectMeasuresIndexer; import org.sonar.server.test.index.TestDoc; import org.sonar.server.test.index.TestIndexDefinition; import org.sonar.server.test.index.TestIndexer; @@ -100,7 +101,10 @@ public class BulkDeleteActionTest { new ComponentCleanerService(dbClient, new IssueAuthorizationIndexer(dbClient, es.client()), new IssueIndexer(dbClient, es.client()), - new TestIndexer(dbClient, es.client()), mockResourceTypes, new ComponentFinder(dbClient)), + new TestIndexer(dbClient, es.client()), + new ProjectMeasuresIndexer(dbClient, es.client()), + mockResourceTypes, + new ComponentFinder(dbClient)), dbClient, userSessionRule))); userSessionRule.setGlobalPermissions(GlobalPermissions.SYSTEM_ADMIN); diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java index cbabaf2fa2e..84275a5f714 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java @@ -49,6 +49,7 @@ import org.sonar.server.issue.index.IssueAuthorizationDoc; import org.sonar.server.issue.index.IssueAuthorizationIndexer; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.project.es.ProjectMeasuresIndexer; import org.sonar.server.test.index.TestDoc; import org.sonar.server.test.index.TestIndexDefinition; import org.sonar.server.test.index.TestIndexer; @@ -102,6 +103,7 @@ public class DeleteActionTest { new IssueAuthorizationIndexer(dbClient, es.client()), new IssueIndexer(dbClient, es.client()), new TestIndexer(dbClient, es.client()), + new ProjectMeasuresIndexer(dbClient, es.client()), mockResourceTypes, new ComponentFinder(dbClient)), new ComponentFinder(dbClient), -- 2.39.5