From: Julien Lancelot Date: Tue, 30 Dec 2014 17:57:46 +0000 (+0100) Subject: SONAR-5849 Performance issue of Project Referentials WS for project with many modules X-Git-Tag: 4.5.2~3 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=769a93d1eaf3d47a6872eb687e7b824accb847c2;p=sonarqube.git SONAR-5849 Performance issue of Project Referentials WS for project with many modules --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectReferentialsLoader.java b/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectReferentialsLoader.java index 1200ec007dc..39ab3e198bd 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectReferentialsLoader.java +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectReferentialsLoader.java @@ -20,7 +20,9 @@ package org.sonar.server.batch; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; import org.sonar.api.ServerComponent; import org.sonar.api.resources.Language; import org.sonar.api.resources.Languages; @@ -28,6 +30,7 @@ import org.sonar.batch.protocol.input.ProjectReferentials; import org.sonar.core.UtcDateUtils; import org.sonar.core.component.AuthorizedComponentDto; import org.sonar.core.component.ComponentDto; +import org.sonar.core.component.ProjectRefentialsComponentDto; import org.sonar.core.permission.GlobalPermissions; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.MyBatis; @@ -75,22 +78,31 @@ public class ProjectReferentialsLoader implements ServerComponent { DbSession session = dbClient.openSession(false); try { ProjectReferentials ref = new ProjectReferentials(); - String projectKey = null; + String projectKey = query.getModuleKey(); AuthorizedComponentDto module = dbClient.componentDao().getNullableAuthorizedComponentByKey(query.getModuleKey(), session); // Current project/module can be null when analysing a new project if (module != null) { ComponentDto project = dbClient.componentDao().getNullableRootProjectByKey(query.getModuleKey(), session); + // Can be null if the given project is a provisioned one if (project != null) { if (!project.key().equals(module.key())) { addSettings(ref, module.getKey(), getSettingsFromParents(module.key(), hasScanPerm, session)); + projectKey = project.key(); + } + + List moduleSettings = dbClient.propertiesDao().selectProjectProperties(query.getModuleKey(), session); + List moduleChildren = dbClient.componentDao().findChildrenModulesFromModule(session, query.getModuleKey()); + List moduleChildrenSettings = newArrayList(); + if (!moduleChildren.isEmpty()) { + moduleChildrenSettings = dbClient.propertiesDao().findChildrenModuleProperties(query.getModuleKey(), session); } - projectKey = project.key(); - addSettingsToChildrenModules(ref, query.getModuleKey(), Maps.newHashMap(), hasScanPerm, session); + TreeModuleSettings treeModuleSettings = new TreeModuleSettings(moduleChildren, moduleChildrenSettings, module, moduleSettings); + + addSettingsToChildrenModules(ref, query.getModuleKey(), Maps.newHashMap(), treeModuleSettings, hasScanPerm, session); } else { // Add settings of the provisioned project addSettings(ref, query.getModuleKey(), getPropertiesMap(dbClient.propertiesDao().selectProjectProperties(query.getModuleKey(), session), hasScanPerm)); - projectKey = query.getModuleKey(); } } @@ -122,15 +134,16 @@ public class ProjectReferentialsLoader implements ServerComponent { } } - private void addSettingsToChildrenModules(ProjectReferentials ref, String projectKey, Map parentProperties, boolean hasScanPerm, DbSession session) { + private void addSettingsToChildrenModules(ProjectReferentials ref, String moduleKey, Map parentProperties, TreeModuleSettings treeModuleSettings, + boolean hasScanPerm, DbSession session) { Map currentParentProperties = newHashMap(); currentParentProperties.putAll(parentProperties); - currentParentProperties.putAll(getPropertiesMap(dbClient.propertiesDao().selectProjectProperties(projectKey, session), hasScanPerm)); - addSettings(ref, projectKey, currentParentProperties); + currentParentProperties.putAll(getPropertiesMap(treeModuleSettings.findModuleSettings(moduleKey), hasScanPerm)); + addSettings(ref, moduleKey, currentParentProperties); - for (ComponentDto module : dbClient.componentDao().findModulesByProject(projectKey, session)) { - addSettings(ref, module.key(), currentParentProperties); - addSettingsToChildrenModules(ref, module.key(), currentParentProperties, hasScanPerm, session); + for (ComponentDto childModule : treeModuleSettings.findChildrenModule(moduleKey)) { + addSettings(ref, childModule.getKey(), currentParentProperties); + addSettingsToChildrenModules(ref, childModule.getKey(), currentParentProperties, treeModuleSettings, hasScanPerm, session); } } @@ -140,7 +153,7 @@ public class ProjectReferentialsLoader implements ServerComponent { } } - private Map getPropertiesMap(List propertyDtos, boolean hasScanPerm) { + private Map getPropertiesMap(List propertyDtos, boolean hasScanPerm) { Map properties = newHashMap(); for (PropertyDto propertyDto : propertyDtos) { String key = propertyDto.getKey(); @@ -219,4 +232,43 @@ public class ProjectReferentialsLoader implements ServerComponent { "Please contact your SonarQube administrator."); } } + + private static class TreeModuleSettings { + + private Map moduleIdsByKey; + private Multimap propertiesByModuleId; + private Multimap moduleChildrenByModuleKey; + + private TreeModuleSettings(List moduleChildren, List moduleChildrenSettings, AuthorizedComponentDto module, List moduleSettings) { + propertiesByModuleId = ArrayListMultimap.create(); + for (PropertyDto settings : moduleChildrenSettings) { + propertiesByModuleId.put(settings.getResourceId(), settings); + } + propertiesByModuleId.putAll(module.getId(), moduleSettings); + moduleIdsByKey = newHashMap(); + for (ProjectRefentialsComponentDto componentDto : moduleChildren) { + moduleIdsByKey.put(componentDto.key(), componentDto.getId()); + } + moduleIdsByKey.put(module.key(), module.getId()); + moduleChildrenByModuleKey = ArrayListMultimap.create(); + for (ProjectRefentialsComponentDto componentDto : moduleChildren) { + String parentModuleKey = componentDto.getParentModuleKey(); + if (parentModuleKey != null) { + moduleChildrenByModuleKey.put(parentModuleKey, componentDto); + } else { + moduleChildrenByModuleKey.put(module.key(), componentDto); + } + } + } + + private List findModuleSettings(String moduleKey) { + Long moduleId = moduleIdsByKey.get(moduleKey); + return newArrayList(propertiesByModuleId.get(moduleId)); + } + + private List findChildrenModule(String moduleKey) { + return newArrayList(moduleChildrenByModuleKey.get(moduleKey)); + } + + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/persistence/ComponentDao.java b/server/sonar-server/src/main/java/org/sonar/server/component/persistence/ComponentDao.java index edcbe529d16..d866a4d1763 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/persistence/ComponentDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/persistence/ComponentDao.java @@ -23,6 +23,7 @@ import org.sonar.api.ServerComponent; import org.sonar.api.utils.System2; import org.sonar.core.component.AuthorizedComponentDto; import org.sonar.core.component.ComponentDto; +import org.sonar.core.component.ProjectRefentialsComponentDto; import org.sonar.core.component.db.ComponentMapper; import org.sonar.core.persistence.DaoComponent; import org.sonar.core.persistence.DbSession; @@ -30,6 +31,7 @@ import org.sonar.server.db.BaseDao; import org.sonar.server.exceptions.NotFoundException; import javax.annotation.CheckForNull; + import java.util.List; /** @@ -84,6 +86,10 @@ public class ComponentDao extends BaseDao return mapper(session).findModulesByProject(projectKey); } + public List findChildrenModulesFromModule(DbSession session, String moduleKey) { + return mapper(session).findChildrenModulesFromModule(moduleKey); + } + @CheckForNull public AuthorizedComponentDto getNullableAuthorizedComponentById(Long id, DbSession session) { return mapper(session).selectAuthorizedComponentById(id); diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/persistence/SnapshotDao.java b/server/sonar-server/src/main/java/org/sonar/server/component/persistence/SnapshotDao.java index 0d2db6552b0..2f3060b9109 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/persistence/SnapshotDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/persistence/SnapshotDao.java @@ -66,6 +66,13 @@ public class SnapshotDao extends BaseDao impl return mapper(session).selectSnapshotAndChildrenOfScope(snapshot.getId(), Scopes.PROJECT); } + /** + * Return all snapshots children (not returning itself) from a module key + */ + public List findChildrenModulesFromModule(DbSession session, String moduleKey) { + return mapper(session).selectChildrenModulesFromModule(moduleKey); + } + public int updateSnapshotAndChildrenLastFlagAndStatus(DbSession session, SnapshotDto snapshot, boolean isLast, String status) { Long rootId = snapshot.getId(); String path = snapshot.getPath() + snapshot.getId() + ".%"; diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/SnapshotTesting.java b/server/sonar-server/src/test/java/org/sonar/server/component/SnapshotTesting.java index 8785d3db8b8..1b174c9a11f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/SnapshotTesting.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/SnapshotTesting.java @@ -24,6 +24,8 @@ import org.sonar.api.utils.DateUtils; import org.sonar.core.component.ComponentDto; import org.sonar.core.component.SnapshotDto; +import java.util.Date; + public class SnapshotTesting { /** @@ -39,7 +41,9 @@ public class SnapshotTesting { .setQualifier(component.qualifier()) .setScope(component.scope()) .setParentId(parentSnapshot.getId()) - .setLast(true); + .setPath(parentSnapshot.getPath() == null ? Long.toString(parentSnapshot.getId()) + "." : parentSnapshot.getPath() + Long.toString(parentSnapshot.getId()) + ".") + .setLast(true) + .setBuildDate(new Date()); } public static SnapshotDto createForProject(ComponentDto project) { @@ -49,7 +53,9 @@ public class SnapshotTesting { .setStatus(SnapshotDto.STATUS_PROCESSED) .setQualifier(project.qualifier()) .setScope(project.scope()) - .setLast(true); + .setPath("") + .setLast(true) + .setBuildDate(new Date()); } public static SnapshotDto defaultSnapshot() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/persistence/ComponentDaoTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/persistence/ComponentDaoTest.java index 5296d905c94..5c58dc4fae7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/persistence/ComponentDaoTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/persistence/ComponentDaoTest.java @@ -26,6 +26,7 @@ import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; import org.sonar.core.component.AuthorizedComponentDto; import org.sonar.core.component.ComponentDto; +import org.sonar.core.component.ProjectRefentialsComponentDto; import org.sonar.core.persistence.AbstractDaoTestCase; import org.sonar.core.persistence.DbSession; import org.sonar.server.exceptions.NotFoundException; @@ -230,6 +231,27 @@ public class ComponentDaoTest extends AbstractDaoTestCase { dao.getAuthorizedComponentByKey("unknown", session); } + @Test + public void find_children_modules_from_module() throws Exception { + setupData("multi-modules"); + + // From root project + List modules = dao.findChildrenModulesFromModule(session, "org.struts:struts"); + assertThat(modules).hasSize(2); + assertThat(modules).onProperty("id").containsOnly(2L, 3L); + assertThat(modules).onProperty("parentModuleKey").containsOnly("org.struts:struts", "org.struts:struts-core"); + + // From module + modules = dao.findChildrenModulesFromModule(session, "org.struts:struts-core"); + assertThat(modules).hasSize(1); + assertThat(modules).onProperty("id").containsOnly(3L); + assertThat(modules).onProperty("parentModuleKey").containsOnly("org.struts:struts-core"); + + // From sub module + modules = dao.findChildrenModulesFromModule(session, "org.struts:struts-data"); + assertThat(modules).isEmpty(); + } + @Test public void insert() { when(system2.now()).thenReturn(DateUtils.parseDate("2014-06-18").getTime()); diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/persistence/SnapshotDaoTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/persistence/SnapshotDaoTest.java index e77aa0ac088..4780e00324c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/persistence/SnapshotDaoTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/persistence/SnapshotDaoTest.java @@ -212,6 +212,26 @@ public class SnapshotDaoTest extends AbstractDaoTestCase { assertThat(snapshots).onProperty("last").containsOnly(false); } + @Test + public void find_children_modules() { + setupData("modules"); + + // From root project + List snapshots = sut.findChildrenModulesFromModule(session, "org.struts:struts"); + assertThat(snapshots).hasSize(2); + assertThat(snapshots).onProperty("id").containsOnly(2L, 3L); + assertThat(snapshots).onProperty("last").containsOnly(true); + + // From module + snapshots = sut.findChildrenModulesFromModule(session, "org.struts:struts-core"); + assertThat(snapshots).hasSize(1); + assertThat(snapshots).onProperty("id").containsOnly(3L); + + // From sub module + snapshots = sut.findChildrenModulesFromModule(session, "org.struts:struts-data"); + assertThat(snapshots).isEmpty(); + } + @Test public void is_last_snapshot_when_no_previous_snapshot() { SnapshotDto snapshot = defaultSnapshot(); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/component/persistence/SnapshotDaoTest/modules.xml b/server/sonar-server/src/test/resources/org/sonar/server/component/persistence/SnapshotDaoTest/modules.xml new file mode 100644 index 00000000000..2e13e2aff7c --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/component/persistence/SnapshotDaoTest/modules.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sonar-core/src/main/java/org/sonar/core/component/ProjectRefentialsComponentDto.java b/sonar-core/src/main/java/org/sonar/core/component/ProjectRefentialsComponentDto.java new file mode 100644 index 00000000000..e531993ce5d --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/component/ProjectRefentialsComponentDto.java @@ -0,0 +1,37 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.component; + +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + +public class ProjectRefentialsComponentDto extends ComponentDto { + + private String parentModuleKey; + + @CheckForNull + public String getParentModuleKey() { + return parentModuleKey; + } + + public void setParentModuleKey(@Nullable String parentModuleKey) { + this.parentModuleKey = parentModuleKey; + } +} diff --git a/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java b/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java index 58cfdba2b5d..7c78b798cba 100644 --- a/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java @@ -22,6 +22,7 @@ package org.sonar.core.component.db; import org.apache.ibatis.annotations.Param; import org.sonar.core.component.AuthorizedComponentDto; import org.sonar.core.component.ComponentDto; +import org.sonar.core.component.ProjectRefentialsComponentDto; import javax.annotation.CheckForNull; @@ -49,6 +50,11 @@ public interface ComponentMapper { */ List findModulesByProject(@Param("projectKey") String projectKey); + /** + * Return all modules children (not returning itself) from a module key + */ + List findChildrenModulesFromModule(@Param("moduleKey") String moduleKey); + long countById(long id); @CheckForNull diff --git a/sonar-core/src/main/java/org/sonar/core/component/db/SnapshotMapper.java b/sonar-core/src/main/java/org/sonar/core/component/db/SnapshotMapper.java index ba98cfb11fb..608a4b99c9c 100644 --- a/sonar-core/src/main/java/org/sonar/core/component/db/SnapshotMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/component/db/SnapshotMapper.java @@ -43,6 +43,8 @@ public interface SnapshotMapper { List selectSnapshotAndChildrenOfScope(@Param(value = "snapshot") Long resourceId, @Param(value = "scope") String scope); + List selectChildrenModulesFromModule(@Param(value = "moduleKey") String moduleKey); + int updateSnapshotAndChildrenLastFlagAndStatus(@Param(value = "root") Long rootId, @Param(value = "pathRootId") Long pathRootId, @Param(value = "path") String path, @Param(value = "isLast") boolean isLast, @Param(value = "status") String status); diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java b/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java index 0c348064e44..98602acfa10 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java @@ -36,6 +36,7 @@ import org.sonar.core.activity.db.ActivityMapper; import org.sonar.core.cluster.WorkQueue; import org.sonar.core.component.AuthorizedComponentDto; import org.sonar.core.component.ComponentDto; +import org.sonar.core.component.ProjectRefentialsComponentDto; import org.sonar.core.component.db.ComponentMapper; import org.sonar.core.component.db.SnapshotMapper; import org.sonar.core.config.Logback; @@ -107,6 +108,7 @@ public class MyBatis implements BatchComponent, ServerComponent { loadAlias(conf, "ActiveDashboard", ActiveDashboardDto.class); loadAlias(conf, "Author", AuthorDto.class); loadAlias(conf, "Component", ComponentDto.class); + loadAlias(conf, "ProjectRefentialsComponent", ProjectRefentialsComponentDto.class); loadAlias(conf, "AuthorizedComponent", AuthorizedComponentDto.class); loadAlias(conf, "Dashboard", DashboardDto.class); loadAlias(conf, "Dependency", DependencyDto.class); diff --git a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java index e7a13b39f26..11c4a80dd0f 100644 --- a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java +++ b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java @@ -108,6 +108,10 @@ public class PropertiesDao implements BatchComponent, ServerComponent, DaoCompon } } + public List findChildrenModuleProperties(String moduleKey, SqlSession session) { + return session.getMapper(PropertiesMapper.class).selectChildrenModuleProperties(moduleKey); + } + public PropertyDto selectProjectProperty(long resourceId, String propertyKey) { SqlSession session = mybatis.openSession(false); PropertiesMapper mapper = session.getMapper(PropertiesMapper.class); diff --git a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java index 6cdf28160af..fe53da10a6a 100644 --- a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java @@ -41,6 +41,8 @@ public interface PropertiesMapper { List selectByQuery(@Param("query") PropertyQuery query); + List selectChildrenModuleProperties(@Param("moduleKey") String moduleKey); + void update(PropertyDto property); void insert(PropertyDto property); diff --git a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml index 452bebb9fcf..3d91df7d665 100644 --- a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml @@ -101,6 +101,15 @@ + + (kee, name, long_name, qualifier, scope, language, root_id, path, created_at) diff --git a/sonar-core/src/main/resources/org/sonar/core/component/db/SnapshotMapper.xml b/sonar-core/src/main/resources/org/sonar/core/component/db/SnapshotMapper.xml index c0112194bcf..0aeb697a2c3 100644 --- a/sonar-core/src/main/resources/org/sonar/core/component/db/SnapshotMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/component/db/SnapshotMapper.xml @@ -68,6 +68,33 @@ AND (s.id = #{snapshot} or s.root_snapshot_id = #{snapshot}) + + + + SELECT + FROM snapshots s + INNER JOIN snapshots root_snapshot ON root_snapshot.id = s.root_snapshot_id AND root_snapshot.islast = ${_true} + INNER JOIN snapshots current_snapshot ON current_snapshot.root_project_id = root_snapshot.project_id AND s.islast = ${_true} + INNER JOIN projects module ON module.id = current_snapshot.project_id AND module.enabled = ${_true} AND module.kee = #{moduleKey} + + AND s.islast = ${_true} + AND s.scope = 'PRJ' + AND + + s.path LIKE current_snapshot.path + CAST(current_snapshot.id AS varchar(15)) + '.%' + + + s.path LIKE concat(current_snapshot.path, current_snapshot.id, '.%') + + + s.path LIKE current_snapshot.path || current_snapshot.id || '.%' + + + + + (parent_snapshot_id, root_snapshot_id, root_project_id, project_id, created_at, build_date, status, purge_status, islast, scope, qualifier, version, path, depth, diff --git a/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml b/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml index 250b49a361b..416b5c1f1eb 100644 --- a/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml @@ -46,6 +46,14 @@ #{propKey} + +