From: Julien Lancelot Date: Tue, 27 Jan 2015 09:17:26 +0000 (+0100) Subject: SONAR-6085 Create Views Index X-Git-Tag: latest-silver-master-#65~61 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=09876321b90335afa1e19b798487853df74e2175;p=sonarqube.git SONAR-6085 Create Views Index --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryLoader.java b/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryLoader.java index c4fd56d2bfe..a4e93028652 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryLoader.java +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryLoader.java @@ -94,15 +94,15 @@ public class ProjectRepositoryLoader implements ServerComponent { projectKey = project.key(); } - List moduleChildren = dbClient.componentDao().findChildrenModulesFromModule(session, query.getModuleKey()); - Map moduleUuidsByKey = moduleUuidsByKey(module, moduleChildren); - Map moduleIdsByKey = moduleIdsByKey(module, moduleChildren); + List modulesTree = dbClient.componentDao().selectModulesTree(session, module.uuid()); + Map moduleUuidsByKey = moduleUuidsByKey(module, modulesTree); + Map moduleIdsByKey = moduleIdsByKey(module, modulesTree); - List moduleChildrenSettings = dbClient.propertiesDao().findChildrenModuleProperties(query.getModuleKey(), session); - TreeModuleSettings treeModuleSettings = new TreeModuleSettings(moduleUuidsByKey, moduleIdsByKey, moduleChildren, moduleChildrenSettings, module); + List modulesTreeSettings = dbClient.propertiesDao().selectModulePropertiesTree(module.uuid(), session); + TreeModuleSettings treeModuleSettings = new TreeModuleSettings(moduleUuidsByKey, moduleIdsByKey, modulesTree, modulesTreeSettings, module); addSettingsToChildrenModules(ref, query.getModuleKey(), Maps.newHashMap(), treeModuleSettings, hasScanPerm, session); - addFileData(session, ref, moduleChildren, module.key()); + addFileData(session, ref, modulesTree, module.uuid()); } addProfiles(ref, projectKey, query.getProfileName(), session); @@ -251,7 +251,7 @@ public class ProjectRepositoryLoader implements ServerComponent { moduleKeysByUuid.put(module.uuid(), module.key()); } - for (FilePathWithHashDto file : dbClient.componentDao().findFilesFromModule(session, moduleKey)) { + for (FilePathWithHashDto file : dbClient.componentDao().selectModuleFilesTree(session, moduleKey)) { FileData fileData = new FileData(file.getSrcHash(), false, null, null, null); ref.addFileData(moduleKeysByUuid.get(file.getModuleUuid()), file.getPath(), fileData); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java b/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java index d91d1d6a71a..39b5688a0a3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/db/ComponentDao.java @@ -22,6 +22,7 @@ package org.sonar.server.component.db; import com.google.common.collect.Lists; import org.sonar.api.ServerComponent; +import org.sonar.api.resources.Qualifiers; import org.sonar.api.resources.Scopes; import org.sonar.api.utils.System2; import org.sonar.core.component.ComponentDto; @@ -37,6 +38,7 @@ import javax.annotation.CheckForNull; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import static com.google.common.collect.Lists.newArrayList; @@ -94,12 +96,12 @@ public class ComponentDao extends BaseDao return mapper(session).findSubProjectsByComponentUuids(keys); } - public List findChildrenModulesFromModule(DbSession session, String moduleKey) { - return mapper(session).findChildrenModulesFromModule(moduleKey, Scopes.PROJECT); + public List selectModulesTree(DbSession session, String rootComponentUuid) { + return mapper(session).selectModulesTree(rootComponentUuid, Scopes.PROJECT); } - public List findFilesFromModule(DbSession session, String moduleKey) { - return mapper(session).findFilesFromModule(moduleKey, Scopes.FILE); + public List selectModuleFilesTree(DbSession session, String rootComponentUuid) { + return mapper(session).selectModuleFilesTree(rootComponentUuid, Scopes.FILE); } public List getByUuids(DbSession session, Collection uuids) { @@ -140,4 +142,12 @@ public class ComponentDao extends BaseDao public List findProjectUuids(DbSession session) { return mapper(session).findProjectUuids(); } + + public List> selectAllViewsAndSubViews(DbSession session) { + return mapper(session).selectAllViewsAndSubViews(Qualifiers.VIEW, Qualifiers.SUBVIEW); + } + + public List selectProjectsFromView(DbSession session, String projectViewUuid, String viewUuid) { + return mapper(session).selectProjectsFromView(projectViewUuid, "%." + viewUuid + ".%", Qualifiers.SUBVIEW); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index 9bedf2b28f7..be14410d363 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -192,6 +192,8 @@ import org.sonar.server.user.ws.FavoritesWs; import org.sonar.server.user.ws.UserPropertiesWs; import org.sonar.server.user.ws.UsersWs; import org.sonar.server.util.*; +import org.sonar.server.view.index.ViewIndexDefinition; +import org.sonar.server.view.index.ViewIndexer; import org.sonar.server.ws.ListingWs; import org.sonar.server.ws.WebServiceEngine; @@ -509,6 +511,10 @@ class ServerComponents { pico.addSingleton(EventsWs.class); pico.addSingleton(ComponentCleanerService.class); + // views + pico.addSingleton(ViewIndexDefinition.class); + pico.addSingleton(ViewIndexer.class); + // issues pico.addSingleton(IssueIndexDefinition.class); pico.addSingleton(IssueIndexer.class); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index dcff7d62a14..1afa9eb55b1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -31,6 +31,7 @@ import org.sonar.server.qualityprofile.index.ActiveRuleIndex; import org.sonar.server.rule.index.RuleIndex; import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.user.index.UserIndexer; +import org.sonar.server.view.index.ViewIndexer; import java.util.Date; @@ -44,15 +45,17 @@ public class IndexSynchronizer { private final IssueAuthorizationIndexer issueAuthorizationIndexer; private final IssueIndexer issueIndexer; private final UserIndexer userIndexer; + private final ViewIndexer viewIndexer; public IndexSynchronizer(DbClient db, IndexClient index, SourceLineIndexer sourceLineIndexer, - IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, UserIndexer userIndexer) { + IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, UserIndexer userIndexer, ViewIndexer viewIndexer) { this.db = db; this.index = index; this.sourceLineIndexer = sourceLineIndexer; this.issueAuthorizationIndexer = issueAuthorizationIndexer; this.issueIndexer = issueIndexer; this.userIndexer = userIndexer; + this.viewIndexer = viewIndexer; } public void execute() { @@ -75,6 +78,9 @@ public class IndexSynchronizer { LOG.info("Index users"); userIndexer.index(); + + LOG.info("Index views"); + viewIndexer.index(); } void synchronize(DbSession session, Dao dao, Index index) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java index d4b655c1001..6934ae53ba5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java @@ -80,7 +80,7 @@ class UserResultSetIterator extends ResultSetIterator { @Override protected UserDoc read(ResultSet rs) throws SQLException { - UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(30)); + UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(7)); String login = rs.getString(1); diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewDoc.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewDoc.java new file mode 100644 index 00000000000..55ba66a1867 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewDoc.java @@ -0,0 +1,52 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.view.index; + +import org.sonar.server.search.BaseDoc; + +import java.util.List; +import java.util.Map; + +public class ViewDoc extends BaseDoc { + + public ViewDoc(Map fields) { + super(fields); + } + + public String uuid() { + return getField(ViewIndexDefinition.FIELD_UUID); + } + + public List projects() { + return getField(ViewIndexDefinition.FIELD_PROJECTS); + } + + public ViewDoc setUuid(String s) { + setField(ViewIndexDefinition.FIELD_UUID, s); + return this; + } + + public ViewDoc setProjects(List s) { + setField(ViewIndexDefinition.FIELD_PROJECTS, s); + return this; + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java new file mode 100644 index 00000000000..ce84709be78 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java @@ -0,0 +1,66 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.view.index; + +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.sonar.api.config.Settings; +import org.sonar.process.ProcessConstants; +import org.sonar.server.es.IndexDefinition; +import org.sonar.server.es.NewIndex; + +/** + * Definition of ES index "views", including settings and fields. + */ +public class ViewIndexDefinition implements IndexDefinition { + + public static final String INDEX = "views"; + + public static final String TYPE_VIEW = "view"; + + public static final String FIELD_UUID = "uuid"; + public static final String FIELD_PROJECTS = "projects"; + + private final Settings settings; + + public ViewIndexDefinition(Settings settings) { + this.settings = settings; + } + + @Override + public void define(IndexDefinitionContext context) { + NewIndex index = context.create(INDEX); + + // shards + boolean clusterMode = settings.getBoolean(ProcessConstants.CLUSTER_ACTIVATE); + if (clusterMode) { + index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4); + index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1); + // else keep defaults (one shard) + } + + // type "view" + NewIndex.NewIndexType mapping = index.createType(TYPE_VIEW); + mapping.setAttribute("_id", ImmutableMap.of("path", FIELD_UUID)); + mapping.stringFieldBuilder(FIELD_UUID).build(); + mapping.stringFieldBuilder(FIELD_PROJECTS).build(); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java new file mode 100644 index 00000000000..c8844d122a7 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -0,0 +1,102 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.view.index; + +import com.google.common.collect.Maps; +import org.elasticsearch.action.update.UpdateRequest; +import org.sonar.core.component.ComponentDto; +import org.sonar.core.persistence.DbSession; +import org.sonar.server.db.DbClient; +import org.sonar.server.es.BaseIndexer; +import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.EsClient; + +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Maps.newHashMap; + +public class ViewIndexer extends BaseIndexer { + + private final DbClient dbClient; + private final EsClient esClient; + + public ViewIndexer(DbClient dbClient, EsClient esClient) { + super(esClient, 300, ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW); + this.dbClient = dbClient; + this.esClient = esClient; + } + + @Override + protected long doIndex(long lastUpdatedAt) { + // Index only if index is empty + long count = esClient.prepareCount(ViewIndexDefinition.INDEX).setTypes(ViewIndexDefinition.TYPE_VIEW).get().getCount(); + if (count == 0) { + DbSession dbSession = dbClient.openSession(false); + try { + Map viewAndProjectViewUuidMap = newHashMap(); + for (Map viewsMap : dbClient.componentDao().selectAllViewsAndSubViews(dbSession)) { + viewAndProjectViewUuidMap.put(viewsMap.get("uuid"), viewsMap.get("projectUuid")); + } + index(dbSession, viewAndProjectViewUuidMap); + } finally { + dbSession.close(); + } + } + return 0L; + } + + public void index(String rootViewUuid) { + DbSession dbSession = dbClient.openSession(false); + try { + Map viewAndProjectViewUuidMap = newHashMap(); + for (ComponentDto viewOrSubView : dbClient.componentDao().selectModulesTree(dbSession, rootViewUuid)) { + viewAndProjectViewUuidMap.put(viewOrSubView.uuid(), viewOrSubView.projectUuid()); + } + index(dbSession, viewAndProjectViewUuidMap); + } finally { + dbSession.close(); + } + } + + private void index(DbSession dbSession, Map viewAndProjectViewUuidMap) { + final BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX); + bulk.start(); + for (Map.Entry entry : viewAndProjectViewUuidMap.entrySet()) { + doIndex(dbSession, bulk, entry.getKey(), entry.getValue()); + } + bulk.stop(); + } + + private void doIndex(DbSession dbSession, BulkIndexer bulk, String uuid, String projectUuid) { + List projects = dbClient.componentDao().selectProjectsFromView(dbSession, projectUuid, uuid); + bulk.add(newUpsertRequest(new ViewDoc(Maps.newHashMap()) + .setUuid(uuid) + .setProjects(projects))); + } + + private UpdateRequest newUpsertRequest(ViewDoc doc) { + return new UpdateRequest(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW, doc.uuid()) + .doc(doc.getFields()) + .upsert(doc.getFields()); + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/package-info.java new file mode 100644 index 00000000000..c99ac8f15dc --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/package-info.java @@ -0,0 +1,25 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +@ParametersAreNonnullByDefault +package org.sonar.server.view.index; + +import javax.annotation.ParametersAreNonnullByDefault; + diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java index aaadc8b91fb..248da521479 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/db/ComponentDaoTest.java @@ -20,6 +20,7 @@ package org.sonar.server.component.db; +import com.google.common.collect.ImmutableMap; import org.apache.ibatis.exceptions.PersistenceException; import org.junit.After; import org.junit.Before; @@ -292,55 +293,55 @@ public class ComponentDaoTest extends AbstractDaoTestCase { } @Test - public void find_children_modules_from_module() throws Exception { + public void select_modules_tree() throws Exception { setupData("multi-modules"); // From root project - List modules = dao.findChildrenModulesFromModule(session, "org.struts:struts"); + List modules = dao.selectModulesTree(session, "ABCD"); assertThat(modules).extracting("uuid").containsOnly("ABCD", "EFGH", "FGHI"); // From module - modules = dao.findChildrenModulesFromModule(session, "org.struts:struts-core"); + modules = dao.selectModulesTree(session, "EFGH"); assertThat(modules).extracting("uuid").containsOnly("EFGH", "FGHI"); // From sub module - modules = dao.findChildrenModulesFromModule(session, "org.struts:struts-data"); + modules = dao.selectModulesTree(session, "FGHI"); assertThat(modules).extracting("uuid").containsOnly("FGHI"); // Folder - assertThat(dao.findChildrenModulesFromModule(session, "org.struts:struts-core:src/org/struts")).isEmpty(); - assertThat(dao.findChildrenModulesFromModule(session, "unknown")).isEmpty(); + assertThat(dao.selectModulesTree(session, "GHIJ")).isEmpty(); + assertThat(dao.selectModulesTree(session, "unknown")).isEmpty(); } @Test - public void find_files_from_module() throws Exception { - setupData("find_files_from_module"); + public void select_module_files_tree() throws Exception { + setupData("select_module_files_tree"); // From root project - List files = dao.findFilesFromModule(session, "org.struts:struts"); + List files = dao.selectModuleFilesTree(session, "ABCD"); assertThat(files).extracting("uuid").containsOnly("EFGHI", "HIJK"); assertThat(files).extracting("moduleUuid").containsOnly("EFGH", "FGHI"); assertThat(files).extracting("srcHash").containsOnly("srcEFGHI", "srcHIJK"); assertThat(files).extracting("path").containsOnly("src/org/struts/pom.xml", "src/org/struts/RequestContext.java"); // From module - files = dao.findFilesFromModule(session, "org.struts:struts-core"); + files = dao.selectModuleFilesTree(session, "EFGH"); assertThat(files).extracting("uuid").containsOnly("EFGHI", "HIJK"); assertThat(files).extracting("moduleUuid").containsOnly("EFGH", "FGHI"); assertThat(files).extracting("srcHash").containsOnly("srcEFGHI", "srcHIJK"); assertThat(files).extracting("path").containsOnly("src/org/struts/pom.xml", "src/org/struts/RequestContext.java"); // From sub module - files = dao.findFilesFromModule(session, "org.struts:struts-data"); + files = dao.selectModuleFilesTree(session, "FGHI"); assertThat(files).extracting("uuid").containsOnly("HIJK"); assertThat(files).extracting("moduleUuid").containsOnly("FGHI"); assertThat(files).extracting("srcHash").containsOnly("srcHIJK"); assertThat(files).extracting("path").containsOnly("src/org/struts/RequestContext.java"); // From directory - assertThat(dao.findFilesFromModule(session, "org.struts:struts-core:src/org/struts")).isEmpty(); + assertThat(dao.selectModuleFilesTree(session, "GHIJ")).isEmpty(); - assertThat(dao.findFilesFromModule(session, "unknown")).isEmpty(); + assertThat(dao.selectModuleFilesTree(session, "unknown")).isEmpty(); } @Test @@ -434,4 +435,27 @@ public class ComponentDaoTest extends AbstractDaoTestCase { public void synchronize_after() { dao.synchronizeAfter(session, new Date(0L)); } + + @Test + public void select_views_and_sub_views() { + setupData("shared_views"); + + assertThat(dao.selectAllViewsAndSubViews(session)).contains( + ImmutableMap.of("uuid", "ABCD", "projectUuid", "ABCD"), + ImmutableMap.of("uuid", "EFGH", "projectUuid", "EFGH"), + ImmutableMap.of("uuid", "FGHI", "projectUuid", "EFGH"), + ImmutableMap.of("uuid", "IJKL", "projectUuid", "IJKL") + ); + } + + @Test + public void select_projects_from_view() { + setupData("shared_views"); + + assertThat(dao.selectProjectsFromView(session, "ABCD", "ABCD")).containsExactly("BCDE"); + assertThat(dao.selectProjectsFromView(session, "EFGH", "EFGH")).containsExactly("GHIJ", "HIJK"); + assertThat(dao.selectProjectsFromView(session, "EFGH", "FGHI")).containsExactly("HIJK"); + assertThat(dao.selectProjectsFromView(session, "IJKL", "IJKL")).isEmpty(); + assertThat(dao.selectProjectsFromView(session, "Unknown", "Unknown")).isEmpty(); + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java index 3c0c0285248..932c92f98b5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java @@ -37,13 +37,13 @@ public class UserIndexDefinitionTest { def.define(context); assertThat(context.getIndices()).hasSize(1); - NewIndex issuesIndex = context.getIndices().get("users"); - assertThat(issuesIndex).isNotNull(); - assertThat(issuesIndex.getTypes().keySet()).containsOnly("user"); + NewIndex index = context.getIndices().get("users"); + assertThat(index).isNotNull(); + assertThat(index.getTypes().keySet()).containsOnly("user"); // no cluster by default - assertThat(issuesIndex.getSettings().get("index.number_of_shards")).isEqualTo("1"); - assertThat(issuesIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("1"); + assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0"); } @Test @@ -53,8 +53,8 @@ public class UserIndexDefinitionTest { UserIndexDefinition def = new UserIndexDefinition(settings); def.define(context); - NewIndex issuesIndex = context.getIndices().get("users"); - assertThat(issuesIndex.getSettings().get("index.number_of_shards")).isEqualTo("4"); - assertThat(issuesIndex.getSettings().get("index.number_of_replicas")).isEqualTo("1"); + NewIndex index = context.getIndices().get("users"); + assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("4"); + assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("1"); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexDefinitionTest.java b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexDefinitionTest.java new file mode 100644 index 00000000000..df48201c599 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexDefinitionTest.java @@ -0,0 +1,61 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.view.index; + +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.process.ProcessConstants; +import org.sonar.server.es.IndexDefinition; +import org.sonar.server.es.NewIndex; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ViewIndexDefinitionTest { + + IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); + + @Test + public void define() throws Exception { + ViewIndexDefinition def = new ViewIndexDefinition(new Settings()); + def.define(context); + + assertThat(context.getIndices()).hasSize(1); + NewIndex index = context.getIndices().get("views"); + assertThat(index).isNotNull(); + assertThat(index.getTypes().keySet()).containsOnly("view"); + + // no cluster by default + assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("1"); + assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + } + + @Test + public void enable_cluster() throws Exception { + Settings settings = new Settings(); + settings.setProperty(ProcessConstants.CLUSTER_ACTIVATE, true); + ViewIndexDefinition def = new ViewIndexDefinition(settings); + def.define(context); + + NewIndex index = context.getIndices().get("views"); + assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("4"); + assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("1"); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java new file mode 100644 index 00000000000..cac123fea8e --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java @@ -0,0 +1,123 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.view.index; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.sonar.api.config.Settings; +import org.sonar.core.persistence.DbTester; +import org.sonar.server.component.db.ComponentDao; +import org.sonar.server.db.DbClient; +import org.sonar.server.es.EsTester; +import org.sonar.test.DbTests; + +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; + +@Category(DbTests.class) +public class ViewIndexerTest { + + @Rule + public DbTester dbTester = new DbTester(); + + @Rule + public EsTester esTester = new EsTester().addDefinitions(new ViewIndexDefinition(new Settings())); + + ViewIndexer indexer; + + @Before + public void setUp() throws Exception { + indexer = new ViewIndexer(new DbClient(dbTester.database(), dbTester.myBatis(), new ComponentDao()), esTester.client()); + } + + @Test + public void index_nothing() throws Exception { + indexer.index(); + assertThat(esTester.countDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW)).isEqualTo(0L); + } + + @Test + public void index() throws Exception { + dbTester.prepareDbUnit(getClass(), "index.xml"); + + indexer.index(); + + List docs = esTester.getDocuments("views", "view", ViewDoc.class); + assertThat(docs).hasSize(4); + + Map viewsByUuid = Maps.uniqueIndex(docs, new Function() { + @Override + public String apply(ViewDoc doc) { + return doc.uuid(); + } + }); + + assertThat(viewsByUuid.get("ABCD").projects()).containsOnly("BCDE"); + assertThat(viewsByUuid.get("EFGH").projects()).containsOnly("GHIJ", "HIJK"); + assertThat(viewsByUuid.get("FGHI").projects()).containsOnly("HIJK"); + assertThat(viewsByUuid.get("IJKL").projects()).isEmpty(); + } + + @Test + public void index_only_if_empty_do_nothing_when_index_already_exists() throws Exception { + // Some views are not in the db + dbTester.prepareDbUnit(getClass(), "index.xml"); + esTester.putDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW, + ImmutableMap.of( + ViewIndexDefinition.FIELD_UUID, "ABCD", + ViewIndexDefinition.FIELD_PROJECTS, newArrayList("BCDE") + )); + + indexer.index(); + + // ... But they shouldn't be indexed + assertThat(esTester.countDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW)).isEqualTo(1L); + } + + @Test + public void index_root_view() throws Exception { + dbTester.prepareDbUnit(getClass(), "index.xml"); + + indexer.index("EFGH"); + + List docs = esTester.getDocuments("views", "view", ViewDoc.class); + assertThat(docs).hasSize(2); + + Map viewsByUuid = Maps.uniqueIndex(docs, new Function() { + @Override + public String apply(ViewDoc doc) { + return doc.uuid(); + } + }); + + assertThat(viewsByUuid.get("EFGH").projects()).containsOnly("GHIJ", "HIJK"); + assertThat(viewsByUuid.get("FGHI").projects()).containsOnly("HIJK"); + } + +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_files_from_module.xml b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_files_from_module.xml deleted file mode 100644 index e1ea8f31504..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/find_files_from_module.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/select_module_files_tree.xml b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/select_module_files_tree.xml new file mode 100644 index 00000000000..e1ea8f31504 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/select_module_files_tree.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/shared_views.xml b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/shared_views.xml new file mode 100644 index 00000000000..03f24d4fb95 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/component/db/ComponentDaoTest/shared_views.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/view/index/ViewIndexerTest/index.xml b/server/sonar-server/src/test/resources/org/sonar/server/view/index/ViewIndexerTest/index.xml new file mode 100644 index 00000000000..03f24d4fb95 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/view/index/ViewIndexerTest/index.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + diff --git a/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java b/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java index 04e4335382a..250b43df518 100644 --- a/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/component/db/ComponentMapper.java @@ -28,6 +28,7 @@ import javax.annotation.CheckForNull; import java.util.Collection; import java.util.List; +import java.util.Map; /** * @since 4.3 @@ -75,14 +76,25 @@ public interface ComponentMapper { List findProjectUuids(); /** - * Return all modules children (including itself) from a module key + * Return all modules tree (child, grand child, etc. including itself) from a module uuid */ - List findChildrenModulesFromModule(@Param("moduleKey") String moduleKey, @Param(value = "scope") String scope); + List selectModulesTree(@Param("moduleUuid") String moduleUuid, @Param(value = "scope") String scope); /** - * Return all files children from a module key + * Return all files children from a module uuid */ - List findFilesFromModule(@Param("moduleKey") String moduleKey, @Param(value = "scope") String scope); + List selectModuleFilesTree(@Param("moduleUuid") String moduleUuid, @Param(value = "scope") String scope); + + /** + * Return all views and sub views + */ + List> selectAllViewsAndSubViews(@Param("viewQualifier") String viewQualifier, @Param("subViewQualifier") String subViewQualifier); + + /** + * Return technical projects from a view or a sub-view + */ + List selectProjectsFromView(@Param("projectViewUuid") String projectViewUuid, @Param("viewUuidLikeQuery") String viewUuidLikeQuery, + @Param("subViewQualifier") String subViewQualifier); long countById(long id); diff --git a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java index ef8d7c3e052..0ee8f546523 100644 --- a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java +++ b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesDao.java @@ -123,8 +123,8 @@ public class PropertiesDao implements BatchComponent, ServerComponent, DaoCompon } } - public List findChildrenModuleProperties(String moduleKey, SqlSession session) { - return session.getMapper(PropertiesMapper.class).selectChildrenModuleProperties(moduleKey, Scopes.PROJECT); + public List selectModulePropertiesTree(String moduleUuid, SqlSession session) { + return session.getMapper(PropertiesMapper.class).selectModulePropertiesTree(moduleUuid, Scopes.PROJECT); } public PropertyDto selectProjectProperty(long resourceId, String propertyKey) { diff --git a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java index 7b38027b823..4124d23e46d 100644 --- a/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/properties/PropertiesMapper.java @@ -44,7 +44,7 @@ public interface PropertiesMapper { List selectByQuery(@Param("query") PropertyQuery query); - List selectChildrenModuleProperties(@Param("moduleKey") String moduleKey, @Param(value = "scope") String scope); + List selectModulePropertiesTree(@Param("moduleUuid") String moduleUuid, @Param(value = "scope") String scope); void update(PropertyDto property); diff --git a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml index d8067bf1215..9edb28eff5c 100644 --- a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml @@ -111,15 +111,15 @@ - SELECT FROM projects p - + - + INNER JOIN projects root_project ON root_project.uuid = p.project_uuid AND root_project.enabled = ${_true} - INNER JOIN projects module ON module.project_uuid = root_project.uuid AND module.enabled = ${_true} AND module.kee = #{moduleKey} + INNER JOIN projects module ON module.project_uuid = root_project.uuid AND module.enabled = ${_true} AND module.uuid = #{moduleUuid} p.enabled = ${_true} AND p.scope = #{scope} @@ -139,11 +139,11 @@ - SELECT p.uuid, p.path, p.module_uuid as moduleUuid, fs.src_hash as srcHash FROM projects p INNER JOIN file_sources fs ON fs.file_uuid=p.uuid - + + + + + (kee, deprecated_kee, uuid, project_uuid, module_uuid, module_uuid_path, name, long_name, qualifier, scope, language, root_id, path, enabled, created_at, authorization_updated_at) diff --git a/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml b/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml index e2b990f507a..40b7eb2a38a 100644 --- a/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/properties/PropertiesMapper.xml @@ -45,10 +45,10 @@ where p.resource_id=#{resourceId} and p.user_id is null - SELECT prop.id as id, prop.prop_key as "key", prop.text_value as value, prop.resource_id as resourceId, prop.user_id as userId FROM properties prop - INNER JOIN (SELECT p.id FROM projects p) modules on modules.id=prop.resource_id + INNER JOIN (SELECT p.id FROM projects p) modules on modules.id=prop.resource_id WHERE prop.user_id IS NULL diff --git a/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java b/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java index aef21c58c11..ab643acf4d6 100644 --- a/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java +++ b/sonar-core/src/test/java/org/sonar/core/properties/PropertiesDaoTest.java @@ -154,23 +154,23 @@ public class PropertiesDaoTest extends AbstractDaoTestCase { } @Test - public void select_children_module_properties() throws Exception { - setupData("select_children_module_properties"); + public void select_module_properties_tree() throws Exception { + setupData("select_module_properties_tree"); - List properties = dao.findChildrenModuleProperties("org.struts:struts", session); + List properties = dao.selectModulePropertiesTree("ABCD", session); assertThat(properties.size(), is(4)); assertThat(properties).extracting("key").containsOnly("struts.one", "core.one", "core.two", "data.one"); assertThat(properties).extracting("value").containsOnly("one", "two"); - properties = dao.findChildrenModuleProperties("org.struts:struts-core", session); + properties = dao.selectModulePropertiesTree("EFGH", session); assertThat(properties.size(), is(3)); assertThat(properties).extracting("key").containsOnly("core.one", "core.two", "data.one"); - properties = dao.findChildrenModuleProperties("org.struts:struts-data", session); + properties = dao.selectModulePropertiesTree("FGHI", session); assertThat(properties.size(), is(1)); assertThat(properties).extracting("key").containsOnly("data.one"); - assertThat(dao.findChildrenModuleProperties("unknown", session).size(), is(0)); + assertThat(dao.selectModulePropertiesTree("unknown", session).size(), is(0)); } @Test diff --git a/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/select_children_module_properties.xml b/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/select_children_module_properties.xml deleted file mode 100644 index 61333a6de4a..00000000000 --- a/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/select_children_module_properties.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/select_module_properties_tree.xml b/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/select_module_properties_tree.xml new file mode 100644 index 00000000000..61333a6de4a --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/core/properties/PropertiesDaoTest/select_module_properties_tree.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +