From cfe5faf0b14841bdad0cae24ac89832c98d1d32a Mon Sep 17 00:00:00 2001 From: Jacek Date: Thu, 12 Aug 2021 16:55:04 +0200 Subject: [PATCH] SONAR-15259 Migrate portfolios from XML to DB --- .../java/org/sonar/ce/queue/CeTaskSubmit.java | 5 + .../java/org/sonar/db/version/SqTables.java | 3 + .../src/main/java/org/sonar/db/DaoModule.java | 2 + .../src/main/java/org/sonar/db/DbClient.java | 7 + .../org/sonar/db/portfolio/PortfolioDao.java | 31 +- .../org/sonar/db/portfolio/PortfolioDto.java | 5 + .../sonar/db/portfolio/PortfolioMapper.java | 8 +- .../sonar/db/portfolio/PortfolioService.java | 212 ------- .../sonar/db/portfolio/PortfolioMapper.xml | 44 +- server/sonar-db-dao/src/schema/schema-sq.ddl | 2 +- .../sonar/db/portfolio/PortfolioDaoTest.java | 26 +- .../sonar/db/component/ComponentDbTester.java | 67 +- .../version/v91/CreatePortfoliosTable.java | 2 +- .../db/migration/version/v91/DbVersion91.java | 1 + .../v91/MigratePortfoliosToNewTables.java | 581 ++++++++++++++++++ .../version/v91/DbVersion91Test.java | 2 +- .../v91/MigratePortfoliosToNewTablesTest.java | 431 +++++++++++++ .../schema.sql | 110 ++++ .../server/component/ComponentUpdater.java | 28 +- 19 files changed, 1321 insertions(+), 246 deletions(-) delete mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioService.java create mode 100644 server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTables.java create mode 100644 server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest.java create mode 100644 server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest/schema.sql diff --git a/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java b/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java index 2cfef65aef5..b37cddb0a7f 100644 --- a/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java +++ b/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java @@ -27,6 +27,7 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import org.sonar.db.component.BranchDto; import org.sonar.db.component.ComponentDto; +import org.sonar.db.portfolio.PortfolioDto; import org.sonar.db.project.ProjectDto; import static com.google.common.base.MoreObjects.firstNonNull; @@ -134,6 +135,10 @@ public final class CeTaskSubmit { return new Component(dto.getUuid(), dto.getUuid()); } + public static Component fromDto(PortfolioDto dto) { + return new Component(dto.getUuid(), dto.getUuid()); + } + public static Component fromDto(BranchDto dto) { return new Component(dto.getUuid(), dto.getProjectUuid()); } diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java index 1102b02a8a7..8ca9b9f9405 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java @@ -72,6 +72,9 @@ public final class SqTables { "perm_templates_groups", "perm_tpl_characteristics", "plugins", + "portfolios", + "portfolio_projects", + "portfolio_references", "projects", "project_alm_settings", "project_branches", diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java index bf6bda78ddb..f288a33053d 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java @@ -58,6 +58,7 @@ import org.sonar.db.permission.UserPermissionDao; import org.sonar.db.permission.template.PermissionTemplateCharacteristicDao; import org.sonar.db.permission.template.PermissionTemplateDao; import org.sonar.db.plugin.PluginDao; +import org.sonar.db.portfolio.PortfolioDao; import org.sonar.db.project.ProjectDao; import org.sonar.db.property.InternalComponentPropertiesDao; import org.sonar.db.property.InternalPropertiesDao; @@ -134,6 +135,7 @@ public class DaoModule extends Module { PermissionTemplateDao.class, PluginDao.class, ProjectDao.class, + PortfolioDao.class, ProjectLinkDao.class, ProjectMappingsDao.class, ProjectQgateAssociationDao.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java index ea6a0516646..3efb4aee142 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java @@ -56,6 +56,7 @@ import org.sonar.db.permission.UserPermissionDao; import org.sonar.db.permission.template.PermissionTemplateCharacteristicDao; import org.sonar.db.permission.template.PermissionTemplateDao; import org.sonar.db.plugin.PluginDao; +import org.sonar.db.portfolio.PortfolioDao; import org.sonar.db.project.ProjectDao; import org.sonar.db.property.InternalComponentPropertiesDao; import org.sonar.db.property.InternalPropertiesDao; @@ -156,6 +157,7 @@ public class DbClient { private final ProjectMappingsDao projectMappingsDao; private final NewCodePeriodDao newCodePeriodDao; private final ProjectDao projectDao; + private final PortfolioDao portfolioDao; private final SessionTokensDao sessionTokensDao; private final SamlMessageIdDao samlMessageIdDao; private final UserDismissedMessagesDao userDismissedMessagesDao; @@ -232,6 +234,7 @@ public class DbClient { internalComponentPropertiesDao = getDao(map, InternalComponentPropertiesDao.class); newCodePeriodDao = getDao(map, NewCodePeriodDao.class); projectDao = getDao(map, ProjectDao.class); + portfolioDao = getDao(map, PortfolioDao.class); sessionTokensDao = getDao(map, SessionTokensDao.class); samlMessageIdDao = getDao(map, SamlMessageIdDao.class); userDismissedMessagesDao = getDao(map, UserDismissedMessagesDao.class); @@ -314,6 +317,10 @@ public class DbClient { return projectDao; } + public PortfolioDao portfolioDao() { + return portfolioDao; + } + public ComponentKeyUpdaterDao componentKeyUpdaterDao() { return componentKeyUpdaterDao; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java index 602ca4b6144..aaa87d1624d 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java @@ -19,15 +19,23 @@ */ package org.sonar.db.portfolio; -import static java.util.Collections.singleton; - +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; import org.sonar.db.Dao; import org.sonar.db.DbSession; +import org.sonar.db.project.ProjectDto; + +import static java.util.Collections.singleton; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toSet; +import static org.sonar.db.DatabaseUtils.executeLargeInputs; public class PortfolioDao implements Dao { private final System2 system2; @@ -54,6 +62,9 @@ public class PortfolioDao implements Dao { } public void insert(DbSession dbSession, PortfolioDto portfolio) { + if (portfolio.getUuid() == null) { + portfolio.setUuid(uuidFactory.create()); + } mapper(dbSession).insert(portfolio); } @@ -63,7 +74,6 @@ public class PortfolioDao implements Dao { mapper(dbSession).deleteProjectsByPortfolioUuids(singleton(portfolioUuid)); } - public void deleteByUuids(DbSession dbSession, Set portfolioUuids) { if (portfolioUuids.isEmpty()) { return; @@ -95,12 +105,16 @@ public class PortfolioDao implements Dao { return mapper(dbSession).selectReferencersByKey(referenceKey); } - public Set getProjects(DbSession dbSession, String portfolioUuid) { + public List getProjects(DbSession dbSession, String portfolioUuid) { return mapper(dbSession).selectProjects(portfolioUuid); } - Set getAllProjectsInHierarchy(DbSession dbSession, String rootUuid) { - return mapper(dbSession).selectAllProjectsInHierarchy(rootUuid); + public Map> getAllProjectsInHierarchy(DbSession dbSession, String rootUuid) { + return mapper(dbSession).selectAllProjectsInHierarchy(rootUuid) + .stream() + .collect(groupingBy( + PortfolioProjectDto::getProjectUuid, + mapping(PortfolioProjectDto::getPortfolioUuid, toSet()))); } public void addProject(DbSession dbSession, String portfolioUuid, String projectUuid) { @@ -123,4 +137,9 @@ public class PortfolioDao implements Dao { public void deleteProjects(DbSession dbSession, String portfolioUuid) { mapper(dbSession).deleteProjects(portfolioUuid); } + + public Map selectKeysByUuids(DbSession dbSession, Collection uuids) { + return executeLargeInputs(uuids, uuids1 -> mapper(dbSession).selectByUuids(uuids1)).stream() + .collect(Collectors.toMap(PortfolioDto::getUuid, PortfolioDto::getKey)); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDto.java index 80ae264af5e..9a06cc38d65 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDto.java @@ -74,6 +74,11 @@ public class PortfolioDto { return this; } + public PortfolioDto setSelectionMode(SelectionMode selectionMode) { + this.selectionMode = selectionMode.name(); + return this; + } + @CheckForNull public String getSelectionExpression() { return selectionExpression; diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java index 929ab51f82e..720ecd99e76 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java @@ -19,10 +19,12 @@ */ package org.sonar.db.portfolio; +import java.util.Collection; import java.util.List; import java.util.Set; import javax.annotation.CheckForNull; import org.apache.ibatis.annotations.Param; +import org.sonar.db.project.ProjectDto; public interface PortfolioMapper { @CheckForNull @@ -53,11 +55,13 @@ public interface PortfolioMapper { List selectReferencersByKey(String referenceKey); - Set selectProjects(String portfolioUuid); + List selectProjects(String portfolioUuid); List selectAllReferencesToPortfolios(); - Set selectAllProjectsInHierarchy(String rootUuid); + Set selectAllProjectsInHierarchy(String rootUuid); + + List selectByUuids(@Param("uuids") Collection uuids); void update(PortfolioDto portfolio); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioService.java b/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioService.java deleted file mode 100644 index 2c16358f579..00000000000 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioService.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (C) 2016-2021 SonarSource SA - * All rights reserved - * mailto:info AT sonarsource DOT com - */ -package org.sonar.db.portfolio; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.project.ProjectDao; -import org.sonar.db.project.ProjectDto; - -import static java.lang.String.format; -import static java.util.Collections.emptyList; - -public class PortfolioService { - private final DbClient dbClient; - private final PortfolioDao portfolioDao; - private final ProjectDao projectDao; - - public PortfolioService(DbClient dbClient, PortfolioDao portfolioDao, ProjectDao projectDao) { - this.dbClient = dbClient; - this.portfolioDao = portfolioDao; - this.projectDao = projectDao; - } - - public void deleteReferencersTo(String referenceKey) { - try (DbSession dbSession = dbClient.openSession(false)) { - List referencers = portfolioDao.selectReferencersByKey(dbSession, referenceKey); - } - } - - public List getRoots() { - try (DbSession dbSession = dbClient.openSession(false)) { - return portfolioDao.selectAllRoots(dbSession); - } - } - - public List getReferencersByKey(String referenceKey) { - try (DbSession dbSession = dbClient.openSession(false)) { - return portfolioDao.selectReferencersByKey(dbSession, referenceKey); - } - } - - public void addReferenceToPortfolio(String portfolioKey, PortfolioDto refPortfolio) { - try (DbSession dbSession = dbClient.openSession(false)) { - PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey); - Set treeReferenceConnections = getTreeReferenceConnections(dbSession, portfolio.getRootUuid()); - if (treeReferenceConnections.contains(refPortfolio.getRootUuid())) { - throw new IllegalArgumentException(format("Can't reference portfolio '%s' in portfolio '%s' - hierarchy would be inconsistent", - refPortfolio.getKey(), portfolio.getKey())); - } - portfolioDao.addReference(dbSession, portfolio.getUuid(), refPortfolio.getUuid()); - } - } - - public void addReferenceToApplication(String portfolioKey, ProjectDto app) { - // TODO - } - - public void refresh(List portfolios) { - Set rootUuids = portfolios.stream().map(PortfolioDto::getRootUuid).collect(Collectors.toSet()); - // TODO eliminate duplicates based on references - - - } - - public void appendProject(String portfolioKey, String projectKey) { - try (DbSession dbSession = dbClient.openSession(false)) { - PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey); - ProjectDto project = projectDao.selectProjectByKey(dbSession, projectKey).orElseThrow(() -> new IllegalArgumentException(format("Project '%s' not found", projectKey))); - - String rootUuid = portfolio.getRootUuid(); - Set allProjectUuids = portfolioDao.getAllProjectsInHierarchy(dbSession, rootUuid); - if (allProjectUuids.contains(project.getUuid())) { - // TODO specify which portfolio? - throw new IllegalArgumentException("The project with key %s is already selected in a portfolio in the hierarchy"); - } - portfolioDao.addProject(dbSession, portfolio.getUuid(), project.getUuid()); - } - } - - public void removeProject(String portfolioKey, String projectKey) { - - } - - public void updateSelectionMode(String portfolioKey, String selectionMode, @Nullable String selectionExpression) { - PortfolioDto.SelectionMode mode = PortfolioDto.SelectionMode.valueOf(selectionMode); - try (DbSession dbSession = dbClient.openSession(false)) { - PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey); - - if (mode.equals(PortfolioDto.SelectionMode.REST)) { - for (PortfolioDto p : portfolioDao.selectTree(dbSession, portfolio.getUuid())) { - if (!p.getUuid().equals(portfolio.getUuid()) && mode.name().equals(portfolio.getSelectionMode())) { - p.setSelectionMode(PortfolioDto.SelectionMode.NONE.name()); - p.setSelectionExpression(null); - portfolioDao.update(dbSession, p); - } - } - } - - portfolio.setSelectionMode(selectionMode); - portfolio.setSelectionExpression(selectionExpression); - - if (!mode.equals(PortfolioDto.SelectionMode.MANUAL)) { - portfolioDao.deleteProjects(dbSession, portfolio.getUuid()); - } - - portfolioDao.update(dbSession, portfolio); - } - } - - /** - * Deletes a portfolio and all of it's children. - * Also deletes references from/to the deleted portfolios and the projects manually assigned to them. - */ - public void delete(String portfolioKey) { - try (DbSession dbSession = dbClient.openSession(false)) { - PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey); - Set subTree = getChildrenRecursively(dbSession, portfolio.getUuid()).stream().map(PortfolioDto::getUuid).collect(Collectors.toSet()); - portfolioDao.deleteByUuids(dbSession, subTree); - // TODO trigger refresh - } - } - - private PortfolioDto selectByKeyOrFail(DbSession dbSession, String portfolioKey) { - return portfolioDao.selectByKey(dbSession, portfolioKey).orElseThrow(() -> new IllegalArgumentException(format("Portfolio '%s' not found", portfolioKey))); - } - - /** - * Gets all portfolios belonging to a subtree, given its root - */ - private List getChildrenRecursively(DbSession dbSession, String portfolioUuid) { - Map tree = portfolioDao.selectTree(dbSession, portfolioUuid).stream().collect(Collectors.toMap(PortfolioDto::getUuid, x -> x)); - Map> childrenMap = new HashMap<>(); - - for (PortfolioDto dto : tree.values()) { - if (dto.getParentUuid() != null) { - childrenMap.computeIfAbsent(dto.getParentUuid(), x -> new HashSet<>()).add(dto.getUuid()); - } - } - - List subTree = new ArrayList<>(); - LinkedList stack = new LinkedList<>(); - stack.add(portfolioUuid); - - while (!stack.isEmpty()) { - Set children = childrenMap.get(stack.removeFirst()); - for (String child : children) { - subTree.add(tree.get(child)); - stack.add(child); - } - } - - return subTree; - } - - /** - * Returns the root UUIDs of all trees with which the tree with the given root uuid is connected through a reference. - * The connection can be incoming or outgoing, and it can be direct or indirect (through other trees). - * - * As an example, let's consider we have the following hierarchies of portfolios: - * - * A D - - -> F - * |\ | |\ - * B C - - - > E G H - * - * Where C references E and D references F. - * All 3 tree roots are connected to all the other roots. - * - * getTreeReferenceConnections(A) would return [D,F] - * getTreeReferenceConnections(D) would return [A,F] - * getTreeReferenceConnections(F) would return [A,D] - */ - private Set getTreeReferenceConnections(DbSession dbSession, String treeRootUuid) { - List references = portfolioDao.selectAllReferencesToPortfolios(dbSession); - - Map> rootConnections = new HashMap<>(); - - for (ReferenceDto ref : references) { - rootConnections.computeIfAbsent(ref.getSourceRootUuid(), x -> new ArrayList<>()).add(ref.getTargetRootUuid()); - rootConnections.computeIfAbsent(ref.getTargetRootUuid(), x -> new ArrayList<>()).add(ref.getSourceRootUuid()); - } - - LinkedList queue = new LinkedList<>(); - Set transitiveReferences = new HashSet<>(); - - // add all direct references - queue.addAll(rootConnections.getOrDefault(treeRootUuid, emptyList())); - - // resolve all transitive references - while (!queue.isEmpty()) { - String uuid = queue.remove(); - if (!transitiveReferences.contains(uuid)) { - queue.addAll(rootConnections.getOrDefault(treeRootUuid, emptyList())); - transitiveReferences.add(uuid); - } - } - - return transitiveReferences; - } - -} diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml index e2a2451f349..663f69e003f 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml @@ -16,6 +16,18 @@ p.updated_at as updatedAt + + p.uuid as uuid, + p.kee as kee, + p.qualifier as qualifier, + p.name as name, + p.description as description, + p.tags as tagsString, + p.private as isPrivate, + p.created_at as createdAt, + p.updated_at as updatedAt + + + + - + SELECT + + FROM portfolio_projects pp + INNER JOIN projects p on p.uuid = pp.project_uuid + WHERE + pp.portfolio_uuid=#{portfolioUuid,jdbcType=VARCHAR} - SELECT - pp.project_uuid + pp.uuid, + pp.project_uuid as projectUuid, + pp.portfolio_uuid as portfolioUuid, + pp.project_uuid as projectUuid, + pp.created_at as createdAt FROM portfolio_projects pp INNER JOIN portfolios p ON pp.portfolio_uuid = p.uuid diff --git a/server/sonar-db-dao/src/schema/schema-sq.ddl b/server/sonar-db-dao/src/schema/schema-sq.ddl index 84adb575198..2913cd77daf 100644 --- a/server/sonar-db-dao/src/schema/schema-sq.ddl +++ b/server/sonar-db-dao/src/schema/schema-sq.ddl @@ -580,7 +580,7 @@ ALTER TABLE "PORTFOLIO_REFERENCES" ADD CONSTRAINT "PK_PORTFOLIO_REFERENCES" PRIM CREATE TABLE "PORTFOLIOS"( "UUID" VARCHAR(40) NOT NULL, - "KEE" VARCHAR(40) NOT NULL, + "KEE" VARCHAR(400) NOT NULL, "NAME" VARCHAR(2000) NOT NULL, "DESCRIPTION" VARCHAR(2000), "ROOT_UUID" VARCHAR(40) NOT NULL, diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java index bc07b4287e5..17cffd59a4a 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java @@ -108,11 +108,10 @@ public class PortfolioDaoTest { @Test public void deleteByUuids() { createPortfolio("p1"); - createPortfolio("p2" ); + createPortfolio("p2"); createPortfolio("p3"); createPortfolio("p4"); - portfolioDao.addProject(db.getSession(), "p1", "proj1"); portfolioDao.addProject(db.getSession(), "p2", "proj1"); @@ -166,23 +165,29 @@ public class PortfolioDaoTest { @Test public void insert_and_select_projects() { + db.components().insertPrivateProject("project1"); + db.components().insertPrivateProject("project2"); + assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).isEmpty(); portfolioDao.addProject(db.getSession(), "portfolio1", "project1"); portfolioDao.addProject(db.getSession(), "portfolio1", "project2"); portfolioDao.addProject(db.getSession(), "portfolio2", "project2"); db.commit(); - assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).containsExactlyInAnyOrder("project1", "project2"); - assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).containsExactlyInAnyOrder("project2"); + assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).extracting(ProjectDto::getUuid).containsExactlyInAnyOrder("project1", "project2"); + assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).extracting(ProjectDto::getUuid).containsExactlyInAnyOrder("project2"); assertThat(portfolioDao.getProjects(db.getSession(), "portfolio3")).isEmpty(); assertThat(db.countRowsOfTable("portfolio_projects")).isEqualTo(3); assertThat(db.select(db.getSession(), "select created_at from portfolio_projects")) .extracting(m -> m.values().iterator().next()) - .containsExactlyInAnyOrder(1L, 2L, 3L); + .containsExactlyInAnyOrder(3L, 4L, 5L); } @Test public void delete_projects() { + db.components().insertPrivateProject("project1"); + db.components().insertPrivateProject("project2"); + assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).isEmpty(); portfolioDao.addProject(db.getSession(), "portfolio1", "project1"); portfolioDao.addProject(db.getSession(), "portfolio1", "project2"); @@ -190,12 +195,17 @@ public class PortfolioDaoTest { portfolioDao.deleteProjects(db.getSession(), "portfolio1"); assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).isEmpty(); - assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).containsExactlyInAnyOrder("project2"); - + assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).extracting(ProjectDto::getUuid) + .containsExactlyInAnyOrder("project2"); } @Test public void getAllProjectsInHierarchy() { + db.components().insertPrivateProject("p1"); + db.components().insertPrivateProject("p2"); + db.components().insertPrivateProject("p3"); + db.components().insertPrivateProject("p4"); + createPortfolio("root", null, "root"); createPortfolio("child1", null, "root"); createPortfolio("child11", "child1", "root"); @@ -207,7 +217,7 @@ public class PortfolioDaoTest { portfolioDao.addProject(db.getSession(), "child11", "p3"); portfolioDao.addProject(db.getSession(), "root2", "p4"); - assertThat(portfolioDao.getAllProjectsInHierarchy(db.getSession(), "root")).containsOnly("p1", "p2", "p3"); + assertThat(portfolioDao.getAllProjectsInHierarchy(db.getSession(), "root").keySet()).containsExactly("p1", "p2", "p3"); assertThat(portfolioDao.getAllProjectsInHierarchy(db.getSession(), "nonexisting")).isEmpty(); } diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/component/ComponentDbTester.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/component/ComponentDbTester.java index 9901e834140..2f330626468 100644 --- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/component/ComponentDbTester.java +++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/component/ComponentDbTester.java @@ -27,12 +27,14 @@ import org.sonar.api.utils.System2; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; +import org.sonar.db.portfolio.PortfolioDto; import org.sonar.db.project.ProjectDto; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.util.Arrays.asList; import static org.sonar.db.component.BranchType.BRANCH; +import static org.sonar.db.portfolio.PortfolioDto.SelectionMode.NONE; public class ComponentDbTester { private final DbTester db; @@ -161,27 +163,67 @@ public class ComponentDbTester { } public final ComponentDto insertPublicPortfolio() { - return insertPublicPortfolio(defaults()); + return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(false), false, defaults(), defaults()); } public final ComponentDto insertPublicPortfolio(String uuid, Consumer dtoPopulator) { - return insertComponentImpl(ComponentTesting.newPortfolio(uuid).setPrivate(false), false, dtoPopulator); + return insertComponentAndPortfolio(ComponentTesting.newPortfolio(uuid).setPrivate(false), false, dtoPopulator, defaults()); } public final ComponentDto insertPublicPortfolio(Consumer dtoPopulator) { - return insertComponentImpl(ComponentTesting.newPortfolio().setPrivate(false), false, dtoPopulator); + return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(false), false, dtoPopulator, defaults()); + } + + public final ComponentDto insertPublicPortfolio(Consumer dtoPopulator, Consumer portfolioPopulator) { + return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(false), false, dtoPopulator, portfolioPopulator); + } + + public ComponentDto insertComponentAndPortfolio(ComponentDto componentDto, boolean isPrivate, Consumer componentPopulator, + Consumer portfolioPopulator) { + insertComponentImpl(componentDto, isPrivate, componentPopulator); + + PortfolioDto portfolioDto = toPortfolioDto(componentDto, System2.INSTANCE.now()); + portfolioPopulator.accept(portfolioDto); + dbClient.portfolioDao().insert(dbSession, portfolioDto); + db.commit(); + return componentDto; } public final ComponentDto insertPrivatePortfolio() { - return insertComponentImpl(ComponentTesting.newPortfolio().setPrivate(true), true, defaults()); + return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(true), true, defaults(), defaults()); } public final ComponentDto insertPrivatePortfolio(String uuid, Consumer dtoPopulator) { - return insertComponentImpl(ComponentTesting.newPortfolio(uuid).setPrivate(true), true, dtoPopulator); + return insertComponentAndPortfolio(ComponentTesting.newPortfolio(uuid).setPrivate(true), true, dtoPopulator, defaults()); } public final ComponentDto insertPrivatePortfolio(Consumer dtoPopulator) { - return insertComponentImpl(ComponentTesting.newPortfolio().setPrivate(true), true, dtoPopulator); + return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(true), true, dtoPopulator, defaults()); + } + + public final ComponentDto insertPrivatePortfolio(Consumer dtoPopulator, Consumer portfolioPopulator) { + return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(true), true, dtoPopulator, portfolioPopulator); + } + + public void addPortfolioProject(ComponentDto portfolio, String... projectUuids) { + for (String uuid : projectUuids) { + dbClient.portfolioDao().addProject(dbSession, portfolio.uuid(), uuid); + } + db.commit(); + } + + public void addPortfolioProject(ComponentDto portfolio, ComponentDto... projects) { + for (ComponentDto project : projects) { + dbClient.portfolioDao().addProject(dbSession, portfolio.uuid(), project.uuid()); + } + db.commit(); + } + + public void addPortfolioProject(PortfolioDto portfolioDto, ProjectDto... projects) { + for (ProjectDto project : projects) { + dbClient.portfolioDao().addProject(dbSession, portfolioDto.getUuid(), project.getUuid()); + } + db.commit(); } public final ComponentDto insertPublicApplication() { @@ -386,6 +428,19 @@ public class ComponentDbTester { .setName(componentDto.name()); } + public static PortfolioDto toPortfolioDto(ComponentDto componentDto, long createTime) { + return new PortfolioDto() + .setUuid(componentDto.uuid()) + .setKey(componentDto.getDbKey()) + .setRootUuid(componentDto.projectUuid()) + .setSelectionMode(NONE.name()) + .setCreatedAt(createTime) + .setUpdatedAt(createTime) + .setPrivate(componentDto.isPrivate()) + .setDescription(componentDto.description()) + .setName(componentDto.name()); + } + private static Consumer defaults() { return t -> { }; diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/CreatePortfoliosTable.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/CreatePortfoliosTable.java index a25d3b1adcf..26d9035fcbc 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/CreatePortfoliosTable.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/CreatePortfoliosTable.java @@ -40,7 +40,7 @@ public class CreatePortfoliosTable extends CreateTableChange { public void execute(Context context, String tableName) throws SQLException { context.execute(new CreateTableBuilder(getDialect(), tableName) .addPkColumn(newVarcharColumnDefBuilder().setColumnName("uuid").setIsNullable(false).setLimit(UUID_SIZE).build()) - .addColumn(newVarcharColumnDefBuilder().setColumnName("kee").setIsNullable(false).setLimit(UUID_SIZE).build()) + .addColumn(newVarcharColumnDefBuilder().setColumnName("kee").setIsNullable(false).setLimit(400).build()) .addColumn(newVarcharColumnDefBuilder().setColumnName("name").setIsNullable(false).setLimit(2000).build()) .addColumn(newVarcharColumnDefBuilder().setColumnName("description").setIsNullable(true).setLimit(2000).build()) .addColumn(newVarcharColumnDefBuilder().setColumnName("root_uuid").setIsNullable(false).setLimit(UUID_SIZE).build()) diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91.java index 45dc9309a74..6a000293a25 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91.java @@ -39,6 +39,7 @@ public class DbVersion91 implements DbVersion { .add(6011, "Create 'portfolios' table", CreatePortfoliosTable.class) .add(6012, "Create 'portfolio_references' table", CreatePortfolioReferencesTable.class) .add(6013, "Create 'portfolio_projects' table", CreatePortfolioProjectsTable.class) + .add(6014, "Migrate portfolios to new tables", MigratePortfoliosToNewTables.class) ; } } diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTables.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTables.java new file mode 100644 index 00000000000..377f7261c28 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTables.java @@ -0,0 +1,581 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v91; + +import com.google.common.collect.Sets; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import javax.xml.XMLConstants; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.transform.sax.SAXSource; +import javax.xml.validation.SchemaFactory; +import org.apache.commons.lang.StringUtils; +import org.codehaus.staxmate.SMInputFactory; +import org.codehaus.staxmate.in.SMHierarchicCursor; +import org.codehaus.staxmate.in.SMInputCursor; +import org.sonar.api.utils.System2; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.Database; +import org.sonar.db.DatabaseUtils; +import org.sonar.server.platform.db.migration.step.DataChange; +import org.sonar.server.platform.db.migration.step.MassUpdate; +import org.sonar.server.platform.db.migration.step.Select; +import org.sonar.server.platform.db.migration.step.Upsert; +import org.sonar.server.platform.db.migration.version.v91.MigratePortfoliosToNewTables.ViewXml.ViewDef; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; +import org.xml.sax.helpers.DefaultHandler; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Optional.ofNullable; +import static javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING; +import static org.apache.commons.io.IOUtils.toInputStream; +import static org.apache.commons.lang.StringUtils.trim; + +public class MigratePortfoliosToNewTables extends DataChange { + private static final String SELECT_DEFAULT_VISIBILITY = "select text_value from properties where prop_key = 'projects.default.visibility'"; + private static final String SELECT_UUID_VISIBILITY_BY_COMPONENT_KEY = "select c.uuid, c.private from components c where c.kee = ?"; + private static final String SELECT_PORTFOLIO_UUID_AND_SELECTION_MODE_BY_KEY = "select uuid,selection_mode from portfolios where kee = ?"; + private static final String SELECT_PROJECT_KEYS_BY_PORTFOLIO_UUID = "select p.kee from portfolio_projects pp " + + "join projects p on pp.project_uuid = p.uuid where pp.portfolio_uuid = ?"; + private static final String SELECT_PROJECT_UUIDS_BY_KEYS = "select p.uuid,p.kee from projects p where p.kee in (PLACEHOLDER)"; + private static final String VIEWS_DEF_KEY = "views.def"; + private static final String PLACEHOLDER = "PLACEHOLDER"; + + private final UuidFactory uuidFactory; + private final System2 system; + + private boolean defaultPrivateFlag; + + public enum SelectionMode { + NONE, MANUAL, REGEXP, REST, TAGS + } + + public MigratePortfoliosToNewTables(Database db, UuidFactory uuidFactory, System2 system) { + super(db); + + this.uuidFactory = uuidFactory; + this.system = system; + } + + @Override + protected void execute(Context context) throws SQLException { + String xml = getViewsDefinition(context); + // skip migration if `views.def` does not exist in the db + if (xml == null) { + return; + } + + this.defaultPrivateFlag = ofNullable(context.prepareSelect(SELECT_DEFAULT_VISIBILITY) + .get(row -> "private".equals(row.getString(1)))) + .orElse(false); + + try { + Map portfolioXmlMap = ViewXml.parse(xml); + List portfolios = new LinkedList<>(portfolioXmlMap.values()); + + Map portfolioDbMap = new HashMap<>(); + for (ViewXml.ViewDef portfolio : portfolios) { + PortfolioDb createdPortfolio = insertPortfolio(context, portfolio); + if (createdPortfolio.selectionMode == SelectionMode.MANUAL) { + insertPortfolioProjects(context, portfolio, createdPortfolio); + } + portfolioDbMap.put(createdPortfolio.kee, createdPortfolio); + } + // all portfolio has been created and new uuids assigned + // update portfolio hierarchy parent/root + insertReferences(context, portfolioXmlMap, portfolioDbMap); + updateHierarchy(context, portfolioXmlMap, portfolioDbMap); + } catch (Exception e) { + throw new IllegalStateException("Failed to migrate views definitions property.", e); + } + } + + private PortfolioDb insertPortfolio(Context context, ViewXml.ViewDef portfolioFromXml) throws SQLException { + long now = system.now(); + PortfolioDb portfolioDb = context.prepareSelect(SELECT_PORTFOLIO_UUID_AND_SELECTION_MODE_BY_KEY) + .setString(1, portfolioFromXml.key) + .get(r -> new PortfolioDb(r.getString(1), portfolioFromXml.key, SelectionMode.valueOf(r.getString(2)))); + + Optional componentDbOpt = ofNullable(context.prepareSelect(SELECT_UUID_VISIBILITY_BY_COMPONENT_KEY) + .setString(1, portfolioFromXml.key) + .get(row -> new ComponentDb(row.getString(1), row.getBoolean(2)))); + + // no portfolio -> insert + if (portfolioDb == null) { + Upsert insertPortfolioQuery = context.prepareUpsert("insert into " + + "portfolios(uuid, kee, private, name, description, root_uuid, parent_uuid, selection_mode, selection_expression, updated_at, created_at) " + + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + + String portfolioUuid = componentDbOpt.map(c -> c.uuid).orElse(uuidFactory.create()); + insertPortfolioQuery.setString(1, portfolioUuid) + .setString(2, portfolioFromXml.key) + .setBoolean(3, componentDbOpt.map(c -> c.visibility).orElse(this.defaultPrivateFlag)) + .setString(4, portfolioFromXml.name) + .setString(5, portfolioFromXml.desc) + .setString(6, PLACEHOLDER) + .setString(7, PLACEHOLDER); + SelectionMode selectionMode = SelectionMode.NONE; + String selectionExpression = null; + if (portfolioFromXml.getProjects() != null && !portfolioFromXml.getProjects().isEmpty()) { + selectionMode = SelectionMode.MANUAL; + } else if (portfolioFromXml.regexp != null && !portfolioFromXml.regexp.isBlank()) { + selectionMode = SelectionMode.REGEXP; + selectionExpression = portfolioFromXml.regexp; + } else if (portfolioFromXml.def) { + selectionMode = SelectionMode.REST; + } else if (portfolioFromXml.tagsAssociation != null && !portfolioFromXml.tagsAssociation.isEmpty()) { + selectionMode = SelectionMode.TAGS; + selectionExpression = String.join(",", portfolioFromXml.tagsAssociation); + } + + insertPortfolioQuery.setString(8, selectionMode.name()) + .setString(9, selectionExpression) + // set dates + .setLong(10, now) + .setLong(11, now) + .execute() + .commit(); + return new PortfolioDb(portfolioUuid, portfolioFromXml.key, selectionMode); + } + return portfolioDb; + } + + private void insertPortfolioProjects(Context context, ViewDef portfolio, PortfolioDb createdPortfolio) throws SQLException { + long now = system.now(); + // select all already added project uuids + Set alreadyAddedPortfolioProjects = new HashSet<>( + context.prepareSelect(SELECT_PROJECT_KEYS_BY_PORTFOLIO_UUID) + .setString(1, createdPortfolio.uuid) + .list(r -> r.getString(1))); + + Set projectKeysFromXml = new HashSet<>(portfolio.getProjects()); + Set projectKeysToBeAdded = Sets.difference(projectKeysFromXml, alreadyAddedPortfolioProjects); + + if (!projectKeysToBeAdded.isEmpty()) { + List projects = findPortfolioProjects(context, projectKeysToBeAdded); + + var upsert = context.prepareUpsert("insert into " + + "portfolio_projects(uuid, portfolio_uuid, project_uuid, created_at) " + + "values (?, ?, ?, ?)"); + for (ProjectDb projectDb : projects) { + upsert.setString(1, uuidFactory.create()) + .setString(2, createdPortfolio.uuid) + .setString(3, projectDb.uuid) + .setLong(4, now) + .addBatch(); + } + if (!projects.isEmpty()) { + upsert.execute() + .commit(); + } + } + } + + private static List findPortfolioProjects(Context context, Set projectKeysToBeAdded) { + return DatabaseUtils.executeLargeInputs(projectKeysToBeAdded, keys -> { + try { + String selectQuery = SELECT_PROJECT_UUIDS_BY_KEYS.replace(PLACEHOLDER, + keys.stream().map(key -> "'" + key + "'").collect( + Collectors.joining(","))); + return context.prepareSelect(selectQuery) + .list(r -> new ProjectDb(r.getString(1), r.getString(2))); + } catch (SQLException e) { + throw new IllegalStateException("Could not execute 'in' query", e); + } + }); + } + + private void insertReferences(Context context, Map portfolioXmlMap, + Map portfolioDbMap) throws SQLException { + Upsert insertQuery = context.prepareUpsert("insert into portfolio_references(uuid, portfolio_uuid, reference_uuid, created_at) values (?, ?, ?, ?)"); + + long now = system.now(); + boolean shouldExecuteQuery = false; + for (ViewDef portfolio : portfolioXmlMap.values()) { + var currentPortfolioUuid = portfolioDbMap.get(portfolio.key).uuid; + Set referencesFromXml = new HashSet<>(portfolio.getReferences()); + Set referencesFromDb = new HashSet<>(context.prepareSelect("select pr.reference_uuid from portfolio_references pr where pr.portfolio_uuid = ?") + .setString(1, currentPortfolioUuid) + .list(row -> row.getString(1))); + + for (String appOrPortfolio : referencesFromXml) { + // if portfolio and hasn't been added already + if (portfolioDbMap.containsKey(appOrPortfolio) && !referencesFromDb.contains(portfolioDbMap.get(appOrPortfolio).uuid)) { + insertQuery + .setString(1, uuidFactory.create()) + .setString(2, currentPortfolioUuid) + .setString(3, portfolioDbMap.get(appOrPortfolio).uuid) + .setLong(4, now) + .addBatch(); + shouldExecuteQuery = true; + } else { + // if application exist and haven't been added + String appUuid = context.prepareSelect("select p.uuid from projects p where p.kee = ?") + .setString(1, appOrPortfolio) + .get(row -> row.getString(1)); + if (appUuid != null && !referencesFromDb.contains(appUuid)) { + insertQuery + .setString(1, uuidFactory.create()) + .setString(2, currentPortfolioUuid) + .setString(3, appUuid) + .setLong(4, now) + .addBatch(); + shouldExecuteQuery = true; + } + } + } + } + if (shouldExecuteQuery) { + insertQuery + .execute() + .commit(); + } + + } + + private static void updateHierarchy(Context context, Map defs, Map portfolioDbMap) throws SQLException { + MassUpdate massUpdate = context.prepareMassUpdate(); + massUpdate.select("select uuid, kee from portfolios where root_uuid = ? or parent_uuid = ?") + .setString(1, PLACEHOLDER) + .setString(2, PLACEHOLDER); + massUpdate.update("update portfolios set root_uuid = ?, parent_uuid = ? where uuid = ?"); + massUpdate.execute((row, update) -> { + String currentPortfolioUuid = row.getString(1); + String currentPortfolioKey = row.getString(2); + + var currentPortfolio = defs.get(currentPortfolioKey); + String parentUuid = ofNullable(currentPortfolio.parent).map(parent -> portfolioDbMap.get(parent).uuid).orElse(null); + String rootUuid = ofNullable(currentPortfolio.root).map(root -> portfolioDbMap.get(root).uuid).orElse(currentPortfolioUuid); + update.setString(1, rootUuid) + .setString(2, parentUuid) + .setString(3, currentPortfolioUuid); + return true; + }); + } + + @CheckForNull + private static String getViewsDefinition(DataChange.Context context) throws SQLException { + Select select = context.prepareSelect("select text_value,clob_value from internal_properties where kee=?"); + select.setString(1, VIEWS_DEF_KEY); + return select.get(row -> { + String v = row.getString(1); + if (v != null) { + return v; + } else { + return row.getString(2); + } + }); + } + + private static class ComponentDb { + String uuid; + boolean visibility; + + public ComponentDb(String uuid, boolean visibility) { + this.uuid = uuid; + this.visibility = visibility; + } + } + + private static class PortfolioDb { + String uuid; + String kee; + SelectionMode selectionMode; + + PortfolioDb(String uuid, String kee, + SelectionMode selectionMode) { + this.uuid = uuid; + this.kee = kee; + this.selectionMode = selectionMode; + } + } + + private static class ProjectDb { + String uuid; + String kee; + + ProjectDb(String uuid, String kee) { + this.uuid = uuid; + this.kee = kee; + } + } + + static class ViewXml { + static final String SCHEMA_VIEWS = "/static/views.xsd"; + static final String VIEWS_HEADER_BARE = ""; + static final Pattern VIEWS_HEADER_BARE_PATTERN = Pattern.compile(VIEWS_HEADER_BARE); + static final String VIEWS_HEADER_FQ = "" + + ""; + + private ViewXml() { + // nothing to do here + } + + private static Map parse(String xml) throws ParserConfigurationException, SAXException, IOException, XMLStreamException { + if (StringUtils.isEmpty(xml)) { + return new LinkedHashMap<>(0); + } + + List views; + validate(xml); + SMInputFactory inputFactory = initStax(); + SMHierarchicCursor rootC = inputFactory.rootElementCursor(new StringReader(xml)); + rootC.advance(); // + SMInputCursor cursor = rootC.childElementCursor(); + views = parseViewDefinitions(cursor); + + Map result = new LinkedHashMap<>(views.size()); + for (ViewDef def : views) { + result.put(def.key, def); + } + + return result; + } + + private static void validate(String xml) throws IOException, SAXException, ParserConfigurationException { + // Replace bare, namespace unaware header with fully qualified header (with schema declaration) + String fullyQualifiedXml = VIEWS_HEADER_BARE_PATTERN.matcher(xml).replaceFirst(VIEWS_HEADER_FQ); + try (InputStream xsd = MigratePortfoliosToNewTables.class.getResourceAsStream(SCHEMA_VIEWS)) { + InputSource viewsDefinition = new InputSource(new InputStreamReader(toInputStream(fullyQualifiedXml, UTF_8), UTF_8)); + + SchemaFactory saxSchemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + + SAXParserFactory parserFactory = SAXParserFactory.newInstance(); + parserFactory.setFeature(FEATURE_SECURE_PROCESSING, true); + parserFactory.setNamespaceAware(true); + parserFactory.setSchema(saxSchemaFactory.newSchema(new SAXSource(new InputSource(xsd)))); + + SAXParser saxParser = parserFactory.newSAXParser(); + saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + saxParser.parse(viewsDefinition, new ViewsValidator()); + } + } + + private static List parseViewDefinitions(SMInputCursor viewsCursor) throws XMLStreamException { + List views = new ArrayList<>(); + while (viewsCursor.getNext() != null) { + ViewDef viewDef = new ViewDef(); + viewDef.setKey(viewsCursor.getAttrValue("key")); + viewDef.setDef(Boolean.parseBoolean(viewsCursor.getAttrValue("def"))); + viewDef.setParent(viewsCursor.getAttrValue("parent")); + viewDef.setRoot(viewsCursor.getAttrValue("root")); + SMInputCursor viewCursor = viewsCursor.childElementCursor(); + while (viewCursor.getNext() != null) { + String nodeName = viewCursor.getLocalName(); + parseChildElement(viewDef, viewCursor, nodeName); + } + views.add(viewDef); + } + return views; + } + + private static void parseChildElement(ViewDef viewDef, SMInputCursor viewCursor, String nodeName) throws XMLStreamException { + if (StringUtils.equals(nodeName, "name")) { + viewDef.setName(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "desc")) { + viewDef.setDesc(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "regexp")) { + viewDef.setRegexp(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "language")) { + viewDef.setLanguage(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "tag_key")) { + viewDef.setTagKey(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "tag_value")) { + viewDef.setTagValue(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "p")) { + viewDef.addProject(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "vw-ref")) { + viewDef.addReference(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "qualifier")) { + viewDef.setQualifier(trim(viewCursor.collectDescendantText())); + } else if (StringUtils.equals(nodeName, "tagsAssociation")) { + parseTagsAssociation(viewDef, viewCursor); + } + } + + private static void parseTagsAssociation(ViewDef def, SMInputCursor viewCursor) throws XMLStreamException { + SMInputCursor projectCursor = viewCursor.childElementCursor(); + while (projectCursor.getNext() != null) { + def.addTagAssociation(trim(projectCursor.collectDescendantText())); + } + } + + private static SMInputFactory initStax() { + XMLInputFactory xmlFactory = XMLInputFactory.newInstance(); + xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE); + xmlFactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, Boolean.FALSE); + // just so it won't try to load DTD in if there's DOCTYPE + xmlFactory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE); + xmlFactory.setProperty(XMLInputFactory.IS_VALIDATING, Boolean.FALSE); + return new SMInputFactory(xmlFactory); + } + + static class ViewDef { + String key = null; + + String parent = null; + + String root = null; + + boolean def = false; + + List p = new ArrayList<>(); + + List vwRef = new ArrayList<>(); + + String name = null; + + String desc = null; + + String regexp = null; + + String language = null; + + String tagKey = null; + + String tagValue = null; + + String qualifier = null; + + Set tagsAssociation = new TreeSet<>(); + + public List getProjects() { + return p; + } + + public List getReferences() { + return vwRef; + } + + public ViewDef setKey(String key) { + this.key = key; + return this; + } + + public ViewDef setParent(String parent) { + this.parent = parent; + return this; + } + + public ViewDef setRoot(@Nullable String root) { + this.root = root; + return this; + } + + public ViewDef setDef(boolean def) { + this.def = def; + return this; + } + + public ViewDef addProject(String project) { + this.p.add(project); + return this; + } + + public ViewDef setName(String name) { + this.name = name; + return this; + } + + public ViewDef setDesc(@Nullable String desc) { + this.desc = desc; + return this; + } + + public ViewDef setRegexp(@Nullable String regexp) { + this.regexp = regexp; + return this; + } + + public ViewDef setLanguage(@Nullable String language) { + this.language = language; + return this; + } + + public ViewDef setTagKey(@Nullable String tagKey) { + this.tagKey = tagKey; + return this; + } + + public ViewDef setTagValue(@Nullable String tagValue) { + this.tagValue = tagValue; + return this; + } + + public ViewDef addReference(String reference) { + this.vwRef.add(reference); + return this; + } + + public ViewDef setQualifier(@Nullable String qualifier) { + this.qualifier = qualifier; + return this; + } + + public ViewDef addTagAssociation(String tag) { + this.tagsAssociation.add(tag); + return this; + } + } + + private static final class ViewsValidator extends DefaultHandler { + @Override + public void error(SAXParseException exception) throws SAXException { + throw exception; + } + + @Override + public void warning(SAXParseException exception) throws SAXException { + throw exception; + } + + @Override + public void fatalError(SAXParseException exception) throws SAXException { + throw exception; + } + } + } +} diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91Test.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91Test.java index e5b51a680b6..0a361304f1b 100644 --- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91Test.java +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91Test.java @@ -41,7 +41,7 @@ public class DbVersion91Test { @Test public void verify_migration_count() { - verifyMigrationCount(underTest, 13); + verifyMigrationCount(underTest, 14); } } diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest.java new file mode 100644 index 00000000000..e8c8e48d9be --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest.java @@ -0,0 +1,431 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v91; + +import java.sql.SQLException; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.impl.utils.TestSystem2; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; +import org.sonar.core.util.UuidFactory; +import org.sonar.core.util.UuidFactoryFast; +import org.sonar.db.CoreDbTester; +import org.sonar.server.platform.db.migration.step.DataChange; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.tuple; + +public class MigratePortfoliosToNewTablesTest { + static final int TEXT_VALUE_MAX_LENGTH = 4000; + private static final long NOW = 10_000_000L; + + @Rule + public final CoreDbTester db = CoreDbTester.createForSchema(MigratePortfoliosToNewTablesTest.class, "schema.sql"); + + private final UuidFactory uuidFactory = UuidFactoryFast.getInstance(); + private final System2 system2 = new TestSystem2().setNow(NOW); + + private final DataChange underTest = new MigratePortfoliosToNewTables(db.database(), uuidFactory, system2); + + private final String SIMPLE_XML_ONE_PORTFOLIO = "" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + private final String SIMPLE_XML_ONLY_PORTFOLIOS = "" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + private final String SIMPLE_XML_PORTFOLIOS_AND_APP = "" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + private final String SIMPLE_XML_PORTFOLIOS_HIERARCHY = "" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " " + + ""; + + private final String SIMPLE_XML_PORTFOLIOS_DIFFERENT_SELECTIONS = "" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " " + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " tag1\n" + + " tag2\n" + + " tag3\n" + + " \n" + + " " + + " \n" + + " \n" + + " \n" + + " \n" + + "

p1-key

\n" + + "

p2-key

\n" + + "

p3-key

\n" + + "

p4-key-not-existing

\n" + + " " + + ""; + + private final String XML_PORTFOLIOS_LEGACY_SELECTIONS = "" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " " + + " \n" + + " \n" + + " \n" + + " \n" + + " tag-key\n" + + " tag-value\n" + + " " + + ""; + + @Test + public void does_not_fail_when_nothing_to_migrate() throws SQLException { + assertThatCode(underTest::execute) + .doesNotThrowAnyException(); + } + + @Test + public void migrate_single_portfolio_with_default_visibility() throws SQLException { + insertViewsDefInternalProperty(SIMPLE_XML_ONE_PORTFOLIO); + insertDefaultVisibilityProperty(true); + insertComponent("uuid", "port2", false); + + underTest.execute(); + // re-entrant + underTest.execute(); + + assertThat(db.select("select kee, private, name, description, " + + "selection_mode, selection_expression, " + + "updated_at, created_at from portfolios")) + .extracting( + row -> row.get("KEE"), + row -> row.get("PRIVATE"), + row -> row.get("NAME"), + row -> row.get("DESCRIPTION"), + row -> row.get("SELECTION_MODE"), + row -> row.get("SELECTION_EXPRESSION"), + row -> row.get("UPDATED_AT"), + row -> row.get("CREATED_AT")) + .containsExactlyInAnyOrder( + tuple("port1", true, "name", "description", "NONE", null, NOW, NOW), + tuple("port2", false, "name2", "description2", "NONE", null, NOW, NOW)); + } + + @Test + public void migrate_portfolios_should_assign_component_uuid() throws SQLException { + insertViewsDefInternalProperty(SIMPLE_XML_ONE_PORTFOLIO); + insertComponent("uuid1", "port1", true); + insertComponent("uuid2", "port2", false); + + underTest.execute(); + // re-entrant + underTest.execute(); + + assertThat(db.select("select uuid, kee, private, name, description, " + + "selection_mode, selection_expression, " + + "updated_at, created_at from portfolios")) + .extracting( + row -> row.get("UUID"), + row -> row.get("KEE"), + row -> row.get("PRIVATE"), + row -> row.get("NAME"), + row -> row.get("DESCRIPTION"), + row -> row.get("SELECTION_MODE"), + row -> row.get("SELECTION_EXPRESSION"), + row -> row.get("UPDATED_AT"), + row -> row.get("CREATED_AT")) + .containsExactlyInAnyOrder( + tuple("uuid1", "port1", true, "name", "description", "NONE", null, NOW, NOW), + tuple("uuid2", "port2", false, "name2", "description2", "NONE", null, NOW, NOW)); + } + + @Test + public void migrate_simple_xml_only_portfolios_references() throws SQLException { + insertViewsDefInternalProperty(SIMPLE_XML_ONLY_PORTFOLIOS); + + underTest.execute(); + // re-entrant + underTest.execute(); + + assertThat(db.select("select kee from portfolios")) + .extracting(row -> row.get("KEE")) + .containsExactlyInAnyOrder("port1", "port2", "port3"); + + var portfolioKeyByUuid = db.select("select uuid, kee from portfolios").stream() + .collect(Collectors.toMap(row -> row.get("KEE"), row -> row.get("UUID").toString())); + + String portfolio1Uuid = portfolioKeyByUuid.get("port1"); + String portfolio2Uuid = portfolioKeyByUuid.get("port2"); + String portfolio3Uuid = portfolioKeyByUuid.get("port3"); + + assertThat(db.select("select portfolio_uuid, reference_uuid from portfolio_references")) + .extracting(row -> row.get("PORTFOLIO_UUID"), row -> row.get("REFERENCE_UUID")) + .containsExactlyInAnyOrder(tuple(portfolio2Uuid, portfolio1Uuid), tuple(portfolio2Uuid, portfolio3Uuid)); + } + + @Test + public void migrate_simple_xml_portfolios_and_apps_references() throws SQLException { + insertViewsDefInternalProperty(SIMPLE_XML_PORTFOLIOS_AND_APP); + + insertProject("app1-uuid", "app1-key", Qualifiers.APP); + insertProject("app2-uuid", "app2-key", Qualifiers.APP); + insertProject("proj1", "proj1-key", Qualifiers.PROJECT); + + underTest.execute(); + // re-entrant + underTest.execute(); + + assertThat(db.select("select kee from portfolios")) + .extracting(row -> row.get("KEE")) + .containsExactlyInAnyOrder("port1", "port2", "port3", "port4"); + + var portfolioKeyByUuid = db.select("select uuid, kee from portfolios").stream() + .collect(Collectors.toMap(row -> row.get("KEE"), row -> row.get("UUID").toString())); + + String portfolio1Uuid = portfolioKeyByUuid.get("port1"); + String portfolio2Uuid = portfolioKeyByUuid.get("port2"); + String portfolio3Uuid = portfolioKeyByUuid.get("port3"); + String portfolio4Uuid = portfolioKeyByUuid.get("port4"); + + assertThat(db.select("select portfolio_uuid, reference_uuid from portfolio_references")) + .extracting(row -> row.get("PORTFOLIO_UUID"), row -> row.get("REFERENCE_UUID")) + .containsExactlyInAnyOrder( + tuple(portfolio2Uuid, portfolio1Uuid), + tuple(portfolio2Uuid, portfolio3Uuid), + tuple(portfolio4Uuid, portfolio2Uuid), + tuple(portfolio4Uuid, "app1-uuid"), + tuple(portfolio4Uuid, "app2-uuid")) + .doesNotContain(tuple(portfolio4Uuid, "app3-key-not-existing")); + } + + @Test + public void migrate_xml_portfolios_hierarchy() throws SQLException { + insertViewsDefInternalProperty(SIMPLE_XML_PORTFOLIOS_HIERARCHY); + + underTest.execute(); + // re-entrant + underTest.execute(); + + assertThat(db.select("select kee from portfolios")) + .extracting(row -> row.get("KEE")) + .containsExactlyInAnyOrder("port1", "port2", "port3", "port4", "port5", "port6", "port7"); + + var portfolioKeyByUuid = db.select("select uuid, kee from portfolios").stream() + .collect(Collectors.toMap(row -> row.get("KEE"), row -> row.get("UUID").toString())); + + String portfolio1Uuid = portfolioKeyByUuid.get("port1"); + String portfolio2Uuid = portfolioKeyByUuid.get("port2"); + String portfolio3Uuid = portfolioKeyByUuid.get("port3"); + String portfolio4Uuid = portfolioKeyByUuid.get("port4"); + String portfolio5Uuid = portfolioKeyByUuid.get("port5"); + String portfolio6Uuid = portfolioKeyByUuid.get("port6"); + String portfolio7Uuid = portfolioKeyByUuid.get("port7"); + + assertThat(db.select("select uuid, parent_uuid, root_uuid from portfolios")) + .extracting(row -> row.get("UUID"), row -> row.get("PARENT_UUID"), row -> row.get("ROOT_UUID")) + .containsExactlyInAnyOrder( + tuple(portfolio1Uuid, null, portfolio1Uuid), + tuple(portfolio2Uuid, portfolio1Uuid, portfolio1Uuid), + tuple(portfolio3Uuid, portfolio1Uuid, portfolio1Uuid), + tuple(portfolio4Uuid, portfolio1Uuid, portfolio1Uuid), + tuple(portfolio5Uuid, portfolio2Uuid, portfolio1Uuid), + tuple(portfolio6Uuid, portfolio3Uuid, portfolio1Uuid), + tuple(portfolio7Uuid, portfolio3Uuid, portfolio1Uuid)); + } + + @Test + public void migrate_xml_portfolios_different_selections() throws SQLException { + insertViewsDefInternalProperty(SIMPLE_XML_PORTFOLIOS_DIFFERENT_SELECTIONS); + + insertProject("p1-uuid", "p1-key", Qualifiers.PROJECT); + insertProject("p2-uuid", "p2-key", Qualifiers.PROJECT); + insertProject("p3-uuid", "p3-key", Qualifiers.PROJECT); + + underTest.execute(); + // re-entrant + underTest.execute(); + + assertThat(db.select("select kee, selection_mode, selection_expression from portfolios")) + .extracting(row -> row.get("KEE"), row -> row.get("SELECTION_MODE"), row -> row.get("SELECTION_EXPRESSION")) + .containsExactlyInAnyOrder( + tuple("port-regexp", "REGEXP", ".*port.*"), + tuple("port-projects", "MANUAL", null), + tuple("port-tags", "TAGS", "tag1,tag2,tag3")); + + // verify projects + assertThat(db.select("select p.kee, pp.project_uuid from " + + "portfolio_projects pp join portfolios p on pp.portfolio_uuid = p.uuid where p.kee = 'port-projects'")) + .extracting(row -> row.get("KEE"), row -> row.get("PROJECT_UUID")) + .containsExactlyInAnyOrder( + tuple("port-projects", "p1-uuid"), + tuple("port-projects", "p2-uuid"), + tuple("port-projects", "p3-uuid")) + .doesNotContain(tuple("port-projects", "p4-uuid-not-existing")); + } + + @Test + public void migrate_xml_portfolios_legacy_selections() throws SQLException { + insertViewsDefInternalProperty(XML_PORTFOLIOS_LEGACY_SELECTIONS); + underTest.execute(); + // re-entrant + underTest.execute(); + + assertThat(db.select("select kee, selection_mode, selection_expression from portfolios")) + .extracting(row -> row.get("KEE"), row -> row.get("SELECTION_MODE"), row -> row.get("SELECTION_EXPRESSION")) + .containsExactlyInAnyOrder( + tuple("port-language", "NONE", null), + tuple("port-tag-value", "NONE", null)); + } + + private void insertProject(String uuid, String key, String qualifier) { + db.executeInsert("PROJECTS", + "UUID", uuid, + "KEE", key, + "QUALIFIER", qualifier, + "ORGANIZATION_UUID", uuid + "-key", + "TAGS", "tag1", + "PRIVATE", Boolean.toString(false), + "UPDATED_AT", System2.INSTANCE.now()); + } + + private void insertViewsDefInternalProperty(@Nullable String xml) { + String valueColumn = "text_value"; + if (xml != null && xml.length() > TEXT_VALUE_MAX_LENGTH) { + valueColumn = "clob_value"; + } + + db.executeInsert("internal_properties", + "kee", "views.def", + "is_empty", "false", + valueColumn, xml, + "created_at", system2.now()); + } + + private void insertComponent(String uuid, String kee, boolean isPrivate) { + db.executeInsert("components", + "uuid", uuid, + "kee", kee, + "enabled", false, + "private", isPrivate, + "root_uuid", uuid, + "uuid_path", uuid, + "project_uuid", uuid); + } + + private void insertDefaultVisibilityProperty(boolean isPrivate) { + db.executeInsert("properties", + "uuid", uuidFactory.create(), + "prop_key", "projects.default.visibility", + "IS_EMPTY", false, + "text_value", isPrivate ? "private" : "public", + "created_at", system2.now()); + } + +} diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest/schema.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest/schema.sql new file mode 100644 index 00000000000..3cd223894dd --- /dev/null +++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest/schema.sql @@ -0,0 +1,110 @@ +CREATE TABLE "PROJECTS"( + "UUID" VARCHAR(40) NOT NULL, + "KEE" VARCHAR(400) NOT NULL, + "QUALIFIER" VARCHAR(10) NOT NULL, + "ORGANIZATION_UUID" VARCHAR(40) NOT NULL, + "NAME" VARCHAR(2000), + "DESCRIPTION" VARCHAR(2000), + "PRIVATE" BOOLEAN NOT NULL, + "TAGS" VARCHAR(500), + "CREATED_AT" BIGINT, + "UPDATED_AT" BIGINT NOT NULL +); +ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID"); +CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE"); +CREATE INDEX "IDX_QUALIFIER" ON "PROJECTS"("QUALIFIER"); + +CREATE TABLE "INTERNAL_PROPERTIES"( + "KEE" VARCHAR(20) NOT NULL, + "IS_EMPTY" BOOLEAN NOT NULL, + "TEXT_VALUE" VARCHAR(4000), + "CLOB_VALUE" CLOB, + "CREATED_AT" BIGINT NOT NULL +); +ALTER TABLE "INTERNAL_PROPERTIES" ADD CONSTRAINT "PK_INTERNAL_PROPERTIES" PRIMARY KEY("KEE"); + +CREATE TABLE "PORTFOLIO_PROJECTS"( + "UUID" VARCHAR(40) NOT NULL, + "PORTFOLIO_UUID" VARCHAR(40) NOT NULL, + "PROJECT_UUID" VARCHAR(40) NOT NULL, + "CREATED_AT" BIGINT NOT NULL +); +ALTER TABLE "PORTFOLIO_PROJECTS" ADD CONSTRAINT "PK_PORTFOLIO_PROJECTS" PRIMARY KEY("UUID"); + +CREATE TABLE "PORTFOLIO_REFERENCES"( + "UUID" VARCHAR(40) NOT NULL, + "PORTFOLIO_UUID" VARCHAR(40) NOT NULL, + "REFERENCE_UUID" VARCHAR(40) NOT NULL, + "CREATED_AT" BIGINT NOT NULL +); +ALTER TABLE "PORTFOLIO_REFERENCES" ADD CONSTRAINT "PK_PORTFOLIO_REFERENCES" PRIMARY KEY("UUID"); + +CREATE TABLE "PORTFOLIOS"( + "UUID" VARCHAR(40) NOT NULL, + "KEE" VARCHAR(400) NOT NULL, + "NAME" VARCHAR(2000) NOT NULL, + "DESCRIPTION" VARCHAR(2000), + "ROOT_UUID" VARCHAR(40) NOT NULL, + "PARENT_UUID" VARCHAR(40), + "PRIVATE" BOOLEAN NOT NULL, + "SELECTION_MODE" VARCHAR(50) NOT NULL, + "SELECTION_EXPRESSION" VARCHAR(50), + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL +); +ALTER TABLE "PORTFOLIOS" ADD CONSTRAINT "PK_PORTFOLIOS" PRIMARY KEY("UUID"); + +CREATE TABLE "PROPERTIES"( + "UUID" VARCHAR(40) NOT NULL, + "PROP_KEY" VARCHAR(512) NOT NULL, + "IS_EMPTY" BOOLEAN NOT NULL, + "TEXT_VALUE" VARCHAR(4000), + "CLOB_VALUE" CLOB, + "CREATED_AT" BIGINT NOT NULL, + "COMPONENT_UUID" VARCHAR(40), + "USER_UUID" VARCHAR(255) +); +ALTER TABLE "PROPERTIES" ADD CONSTRAINT "PK_PROPERTIES" PRIMARY KEY("UUID"); +CREATE INDEX "PROPERTIES_KEY" ON "PROPERTIES"("PROP_KEY"); + +CREATE TABLE "COMPONENTS"( + "UUID" VARCHAR(50) NOT NULL, + "KEE" VARCHAR(400), + "DEPRECATED_KEE" VARCHAR(400), + "NAME" VARCHAR(2000), + "LONG_NAME" VARCHAR(2000), + "DESCRIPTION" VARCHAR(2000), + "ENABLED" BOOLEAN DEFAULT TRUE NOT NULL, + "SCOPE" VARCHAR(3), + "QUALIFIER" VARCHAR(10), + "PRIVATE" BOOLEAN NOT NULL, + "ROOT_UUID" VARCHAR(50) NOT NULL, + "LANGUAGE" VARCHAR(20), + "COPY_COMPONENT_UUID" VARCHAR(50), + "PATH" VARCHAR(2000), + "UUID_PATH" VARCHAR(1500) NOT NULL, + "PROJECT_UUID" VARCHAR(50) NOT NULL, + "MODULE_UUID" VARCHAR(50), + "MODULE_UUID_PATH" VARCHAR(1500), + "MAIN_BRANCH_PROJECT_UUID" VARCHAR(50), + "B_CHANGED" BOOLEAN, + "B_NAME" VARCHAR(500), + "B_LONG_NAME" VARCHAR(500), + "B_DESCRIPTION" VARCHAR(2000), + "B_ENABLED" BOOLEAN, + "B_QUALIFIER" VARCHAR(10), + "B_LANGUAGE" VARCHAR(20), + "B_COPY_COMPONENT_UUID" VARCHAR(50), + "B_PATH" VARCHAR(2000), + "B_UUID_PATH" VARCHAR(1500), + "B_MODULE_UUID" VARCHAR(50), + "B_MODULE_UUID_PATH" VARCHAR(1500), + "CREATED_AT" TIMESTAMP +); +CREATE UNIQUE INDEX "PROJECTS_KEE" ON "COMPONENTS"("KEE"); +CREATE INDEX "PROJECTS_MODULE_UUID" ON "COMPONENTS"("MODULE_UUID"); +CREATE INDEX "PROJECTS_PROJECT_UUID" ON "COMPONENTS"("PROJECT_UUID"); +CREATE INDEX "PROJECTS_QUALIFIER" ON "COMPONENTS"("QUALIFIER"); +CREATE INDEX "PROJECTS_ROOT_UUID" ON "COMPONENTS"("ROOT_UUID"); +CREATE INDEX "PROJECTS_UUID" ON "COMPONENTS"("UUID"); +CREATE INDEX "IDX_MAIN_BRANCH_PRJ_UUID" ON "COMPONENTS"("MAIN_BRANCH_PROJECT_UUID"); diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentUpdater.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentUpdater.java index ef052363271..9da3b7edd93 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentUpdater.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentUpdater.java @@ -36,6 +36,8 @@ import org.sonar.db.DbSession; import org.sonar.db.component.BranchDto; import org.sonar.db.component.BranchType; import org.sonar.db.component.ComponentDto; +import org.sonar.db.portfolio.PortfolioDto; +import org.sonar.db.portfolio.PortfolioDto.SelectionMode; import org.sonar.db.project.ProjectDto; import org.sonar.server.es.ProjectIndexer.Cause; import org.sonar.server.es.ProjectIndexers; @@ -90,7 +92,7 @@ public class ComponentUpdater { * Don't forget to call commitAndIndex(...) when ready to commit. */ public ComponentDto createWithoutCommit(DbSession dbSession, NewComponent newComponent, - @Nullable String userUuid, @Nullable String userLogin, Consumer componentModifier) { + @Nullable String userUuid, @Nullable String userLogin, Consumer componentModifier) { return createWithoutCommit(dbSession, newComponent, userUuid, userLogin, null, componentModifier); } @@ -144,6 +146,12 @@ public class ComponentUpdater { ProjectDto projectDto = toProjectDto(component, now); dbClient.projectDao().insert(session, projectDto); } + + if (isRootView(component)) { + PortfolioDto portfolioDto = toPortfolioDto(component, now); + dbClient.portfolioDao().insert(session, portfolioDto); + } + return component; } @@ -159,11 +167,29 @@ public class ComponentUpdater { .setCreatedAt(now); } + private static PortfolioDto toPortfolioDto(ComponentDto component, long now) { + return new PortfolioDto() + .setUuid(component.uuid()) + .setRootUuid(component.projectUuid()) + .setKey(component.getKey()) + .setName(component.name()) + .setPrivate(component.isPrivate()) + .setDescription(component.description()) + .setSelectionMode(SelectionMode.NONE.name()) + .setUpdatedAt(now) + .setCreatedAt(now); + } + private static boolean isRootProject(ComponentDto componentDto) { return Scopes.PROJECT.equals(componentDto.scope()) && MAIN_BRANCH_QUALIFIERS.contains(componentDto.qualifier()); } + private static boolean isRootView(ComponentDto componentDto) { + return Scopes.PROJECT.equals(componentDto.scope()) + && Qualifiers.VIEW.contains(componentDto.qualifier()); + } + private void createMainBranch(DbSession session, String componentUuid, @Nullable String mainBranch) { BranchDto branch = new BranchDto() .setBranchType(BranchType.BRANCH) -- 2.39.5