mirrors
/
sonarqube
mirror of https://github.com/SonarSource/sonarqube.git
Watch 1
Star 0
Fork 0
Code Issues 0 Releases 237 Wiki Activity
Browse Source

SONAR-15259 Migrate portfolios from XML to DB

tags/9.1.0.47736
Jacek 2 years ago
parent
6ad72a24b2
commit
cfe5faf0b1
19 changed files with 1321 additions and 246 deletions
Split View Show Diff Stats
  1. 5
    0
      server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java
  2. 3
    0
      server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java
  3. 2
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java
  4. 7
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java
  5. 25
    6
      server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java
  6. 5
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDto.java
  7. 6
    2
      server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java
  8. 0
    212
      server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioService.java
  9. 36
    8
      server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml
  10. 1
    1
      server/sonar-db-dao/src/schema/schema-sq.ddl
  11. 18
    8
      server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java
  12. 61
    6
      server/sonar-db-dao/src/testFixtures/java/org/sonar/db/component/ComponentDbTester.java
  13. 1
    1
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/CreatePortfoliosTable.java
  14. 1
    0
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91.java
  15. 581
    0
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTables.java
  16. 1
    1
      server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91Test.java
  17. 431
    0
      server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest.java
  18. 110
    0
      server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest/schema.sql
  19. 27
    1
      server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentUpdater.java

+ 5
- 0
server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java View File

@@ -27,6 +27,7 @@ import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable;
import org.sonar.db.component.BranchDto;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.portfolio.PortfolioDto;
import org.sonar.db.project.ProjectDto;

import static com.google.common.base.MoreObjects.firstNonNull;
@@ -134,6 +135,10 @@ public final class CeTaskSubmit {
return new Component(dto.getUuid(), dto.getUuid());
}

public static Component fromDto(PortfolioDto dto) {
return new Component(dto.getUuid(), dto.getUuid());
}

public static Component fromDto(BranchDto dto) {
return new Component(dto.getUuid(), dto.getProjectUuid());
}

+ 3
- 0
server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java View File

@@ -72,6 +72,9 @@ public final class SqTables {
"perm_templates_groups",
"perm_tpl_characteristics",
"plugins",
"portfolios",
"portfolio_projects",
"portfolio_references",
"projects",
"project_alm_settings",
"project_branches",

+ 2
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java View File

@@ -58,6 +58,7 @@ import org.sonar.db.permission.UserPermissionDao;
import org.sonar.db.permission.template.PermissionTemplateCharacteristicDao;
import org.sonar.db.permission.template.PermissionTemplateDao;
import org.sonar.db.plugin.PluginDao;
import org.sonar.db.portfolio.PortfolioDao;
import org.sonar.db.project.ProjectDao;
import org.sonar.db.property.InternalComponentPropertiesDao;
import org.sonar.db.property.InternalPropertiesDao;
@@ -134,6 +135,7 @@ public class DaoModule extends Module {
PermissionTemplateDao.class,
PluginDao.class,
ProjectDao.class,
PortfolioDao.class,
ProjectLinkDao.class,
ProjectMappingsDao.class,
ProjectQgateAssociationDao.class,

+ 7
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java View File

@@ -56,6 +56,7 @@ import org.sonar.db.permission.UserPermissionDao;
import org.sonar.db.permission.template.PermissionTemplateCharacteristicDao;
import org.sonar.db.permission.template.PermissionTemplateDao;
import org.sonar.db.plugin.PluginDao;
import org.sonar.db.portfolio.PortfolioDao;
import org.sonar.db.project.ProjectDao;
import org.sonar.db.property.InternalComponentPropertiesDao;
import org.sonar.db.property.InternalPropertiesDao;
@@ -156,6 +157,7 @@ public class DbClient {
private final ProjectMappingsDao projectMappingsDao;
private final NewCodePeriodDao newCodePeriodDao;
private final ProjectDao projectDao;
private final PortfolioDao portfolioDao;
private final SessionTokensDao sessionTokensDao;
private final SamlMessageIdDao samlMessageIdDao;
private final UserDismissedMessagesDao userDismissedMessagesDao;
@@ -232,6 +234,7 @@ public class DbClient {
internalComponentPropertiesDao = getDao(map, InternalComponentPropertiesDao.class);
newCodePeriodDao = getDao(map, NewCodePeriodDao.class);
projectDao = getDao(map, ProjectDao.class);
portfolioDao = getDao(map, PortfolioDao.class);
sessionTokensDao = getDao(map, SessionTokensDao.class);
samlMessageIdDao = getDao(map, SamlMessageIdDao.class);
userDismissedMessagesDao = getDao(map, UserDismissedMessagesDao.class);
@@ -314,6 +317,10 @@ public class DbClient {
return projectDao;
}

public PortfolioDao portfolioDao() {
return portfolioDao;
}

public ComponentKeyUpdaterDao componentKeyUpdaterDao() {
return componentKeyUpdaterDao;
}

+ 25
- 6
server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDao.java View File

@@ -19,15 +19,23 @@
*/
package org.sonar.db.portfolio;

import static java.util.Collections.singleton;

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.sonar.api.utils.System2;
import org.sonar.core.util.UuidFactory;
import org.sonar.db.Dao;
import org.sonar.db.DbSession;
import org.sonar.db.project.ProjectDto;

import static java.util.Collections.singleton;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toSet;
import static org.sonar.db.DatabaseUtils.executeLargeInputs;

public class PortfolioDao implements Dao {
private final System2 system2;
@@ -54,6 +62,9 @@ public class PortfolioDao implements Dao {
}

public void insert(DbSession dbSession, PortfolioDto portfolio) {
if (portfolio.getUuid() == null) {
portfolio.setUuid(uuidFactory.create());
}
mapper(dbSession).insert(portfolio);
}

@@ -63,7 +74,6 @@ public class PortfolioDao implements Dao {
mapper(dbSession).deleteProjectsByPortfolioUuids(singleton(portfolioUuid));
}


public void deleteByUuids(DbSession dbSession, Set<String> portfolioUuids) {
if (portfolioUuids.isEmpty()) {
return;
@@ -95,12 +105,16 @@ public class PortfolioDao implements Dao {
return mapper(dbSession).selectReferencersByKey(referenceKey);
}

public Set<String> getProjects(DbSession dbSession, String portfolioUuid) {
public List<ProjectDto> getProjects(DbSession dbSession, String portfolioUuid) {
return mapper(dbSession).selectProjects(portfolioUuid);
}

Set<String> getAllProjectsInHierarchy(DbSession dbSession, String rootUuid) {
return mapper(dbSession).selectAllProjectsInHierarchy(rootUuid);
public Map<String, Set<String>> getAllProjectsInHierarchy(DbSession dbSession, String rootUuid) {
return mapper(dbSession).selectAllProjectsInHierarchy(rootUuid)
.stream()
.collect(groupingBy(
PortfolioProjectDto::getProjectUuid,
mapping(PortfolioProjectDto::getPortfolioUuid, toSet())));
}

public void addProject(DbSession dbSession, String portfolioUuid, String projectUuid) {
@@ -123,4 +137,9 @@ public class PortfolioDao implements Dao {
public void deleteProjects(DbSession dbSession, String portfolioUuid) {
mapper(dbSession).deleteProjects(portfolioUuid);
}

public Map<String, String> selectKeysByUuids(DbSession dbSession, Collection<String> uuids) {
return executeLargeInputs(uuids, uuids1 -> mapper(dbSession).selectByUuids(uuids1)).stream()
.collect(Collectors.toMap(PortfolioDto::getUuid, PortfolioDto::getKey));
}
}

+ 5
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioDto.java View File

@@ -74,6 +74,11 @@ public class PortfolioDto {
return this;
}

public PortfolioDto setSelectionMode(SelectionMode selectionMode) {
this.selectionMode = selectionMode.name();
return this;
}

@CheckForNull
public String getSelectionExpression() {
return selectionExpression;

+ 6
- 2
server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioMapper.java View File

@@ -19,10 +19,12 @@
*/
package org.sonar.db.portfolio;

import java.util.Collection;
import java.util.List;
import java.util.Set;
import javax.annotation.CheckForNull;
import org.apache.ibatis.annotations.Param;
import org.sonar.db.project.ProjectDto;

public interface PortfolioMapper {
@CheckForNull
@@ -53,11 +55,13 @@ public interface PortfolioMapper {

List<PortfolioDto> selectReferencersByKey(String referenceKey);

Set<String> selectProjects(String portfolioUuid);
List<ProjectDto> selectProjects(String portfolioUuid);

List<ReferenceDto> selectAllReferencesToPortfolios();

Set<String> selectAllProjectsInHierarchy(String rootUuid);
Set<PortfolioProjectDto> selectAllProjectsInHierarchy(String rootUuid);

List<PortfolioDto> selectByUuids(@Param("uuids") Collection<String> uuids);

void update(PortfolioDto portfolio);


+ 0
- 212
server/sonar-db-dao/src/main/java/org/sonar/db/portfolio/PortfolioService.java View File

@@ -1,212 +0,0 @@
/*
* Copyright (C) 2016-2021 SonarSource SA
* All rights reserved
* mailto:info AT sonarsource DOT com
*/
package org.sonar.db.portfolio;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.project.ProjectDao;
import org.sonar.db.project.ProjectDto;

import static java.lang.String.format;
import static java.util.Collections.emptyList;

public class PortfolioService {
private final DbClient dbClient;
private final PortfolioDao portfolioDao;
private final ProjectDao projectDao;

public PortfolioService(DbClient dbClient, PortfolioDao portfolioDao, ProjectDao projectDao) {
this.dbClient = dbClient;
this.portfolioDao = portfolioDao;
this.projectDao = projectDao;
}

public void deleteReferencersTo(String referenceKey) {
try (DbSession dbSession = dbClient.openSession(false)) {
List<PortfolioDto> referencers = portfolioDao.selectReferencersByKey(dbSession, referenceKey);
}
}

public List<PortfolioDto> getRoots() {
try (DbSession dbSession = dbClient.openSession(false)) {
return portfolioDao.selectAllRoots(dbSession);
}
}

public List<PortfolioDto> getReferencersByKey(String referenceKey) {
try (DbSession dbSession = dbClient.openSession(false)) {
return portfolioDao.selectReferencersByKey(dbSession, referenceKey);
}
}

public void addReferenceToPortfolio(String portfolioKey, PortfolioDto refPortfolio) {
try (DbSession dbSession = dbClient.openSession(false)) {
PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey);
Set<String> treeReferenceConnections = getTreeReferenceConnections(dbSession, portfolio.getRootUuid());
if (treeReferenceConnections.contains(refPortfolio.getRootUuid())) {
throw new IllegalArgumentException(format("Can't reference portfolio '%s' in portfolio '%s' - hierarchy would be inconsistent",
refPortfolio.getKey(), portfolio.getKey()));
}
portfolioDao.addReference(dbSession, portfolio.getUuid(), refPortfolio.getUuid());
}
}

public void addReferenceToApplication(String portfolioKey, ProjectDto app) {
// TODO
}

public void refresh(List<PortfolioDto> portfolios) {
Set<String> rootUuids = portfolios.stream().map(PortfolioDto::getRootUuid).collect(Collectors.toSet());
// TODO eliminate duplicates based on references


}

public void appendProject(String portfolioKey, String projectKey) {
try (DbSession dbSession = dbClient.openSession(false)) {
PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey);
ProjectDto project = projectDao.selectProjectByKey(dbSession, projectKey).orElseThrow(() -> new IllegalArgumentException(format("Project '%s' not found", projectKey)));

String rootUuid = portfolio.getRootUuid();
Set<String> allProjectUuids = portfolioDao.getAllProjectsInHierarchy(dbSession, rootUuid);
if (allProjectUuids.contains(project.getUuid())) {
// TODO specify which portfolio?
throw new IllegalArgumentException("The project with key %s is already selected in a portfolio in the hierarchy");
}
portfolioDao.addProject(dbSession, portfolio.getUuid(), project.getUuid());
}
}

public void removeProject(String portfolioKey, String projectKey) {

}

public void updateSelectionMode(String portfolioKey, String selectionMode, @Nullable String selectionExpression) {
PortfolioDto.SelectionMode mode = PortfolioDto.SelectionMode.valueOf(selectionMode);
try (DbSession dbSession = dbClient.openSession(false)) {
PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey);

if (mode.equals(PortfolioDto.SelectionMode.REST)) {
for (PortfolioDto p : portfolioDao.selectTree(dbSession, portfolio.getUuid())) {
if (!p.getUuid().equals(portfolio.getUuid()) && mode.name().equals(portfolio.getSelectionMode())) {
p.setSelectionMode(PortfolioDto.SelectionMode.NONE.name());
p.setSelectionExpression(null);
portfolioDao.update(dbSession, p);
}
}
}

portfolio.setSelectionMode(selectionMode);
portfolio.setSelectionExpression(selectionExpression);

if (!mode.equals(PortfolioDto.SelectionMode.MANUAL)) {
portfolioDao.deleteProjects(dbSession, portfolio.getUuid());
}

portfolioDao.update(dbSession, portfolio);
}
}

/**
* Deletes a portfolio and all of it's children.
* Also deletes references from/to the deleted portfolios and the projects manually assigned to them.
*/
public void delete(String portfolioKey) {
try (DbSession dbSession = dbClient.openSession(false)) {
PortfolioDto portfolio = selectByKeyOrFail(dbSession, portfolioKey);
Set<String> subTree = getChildrenRecursively(dbSession, portfolio.getUuid()).stream().map(PortfolioDto::getUuid).collect(Collectors.toSet());
portfolioDao.deleteByUuids(dbSession, subTree);
// TODO trigger refresh
}
}

private PortfolioDto selectByKeyOrFail(DbSession dbSession, String portfolioKey) {
return portfolioDao.selectByKey(dbSession, portfolioKey).orElseThrow(() -> new IllegalArgumentException(format("Portfolio '%s' not found", portfolioKey)));
}

/**
* Gets all portfolios belonging to a subtree, given its root
*/
private List<PortfolioDto> getChildrenRecursively(DbSession dbSession, String portfolioUuid) {
Map<String, PortfolioDto> tree = portfolioDao.selectTree(dbSession, portfolioUuid).stream().collect(Collectors.toMap(PortfolioDto::getUuid, x -> x));
Map<String, Set<String>> childrenMap = new HashMap<>();

for (PortfolioDto dto : tree.values()) {
if (dto.getParentUuid() != null) {
childrenMap.computeIfAbsent(dto.getParentUuid(), x -> new HashSet<>()).add(dto.getUuid());
}
}

List<PortfolioDto> subTree = new ArrayList<>();
LinkedList<String> stack = new LinkedList<>();
stack.add(portfolioUuid);

while (!stack.isEmpty()) {
Set<String> children = childrenMap.get(stack.removeFirst());
for (String child : children) {
subTree.add(tree.get(child));
stack.add(child);
}
}

return subTree;
}

/**
* Returns the root UUIDs of all trees with which the tree with the given root uuid is connected through a reference.
* The connection can be incoming or outgoing, and it can be direct or indirect (through other trees).
*
* As an example, let's consider we have the following hierarchies of portfolios:
*
* A D - - -> F
* |\ | |\
* B C - - - > E G H
*
* Where C references E and D references F.
* All 3 tree roots are connected to all the other roots.
*
* getTreeReferenceConnections(A) would return [D,F]
* getTreeReferenceConnections(D) would return [A,F]
* getTreeReferenceConnections(F) would return [A,D]
*/
private Set<String> getTreeReferenceConnections(DbSession dbSession, String treeRootUuid) {
List<ReferenceDto> references = portfolioDao.selectAllReferencesToPortfolios(dbSession);

Map<String, List<String>> rootConnections = new HashMap<>();

for (ReferenceDto ref : references) {
rootConnections.computeIfAbsent(ref.getSourceRootUuid(), x -> new ArrayList<>()).add(ref.getTargetRootUuid());
rootConnections.computeIfAbsent(ref.getTargetRootUuid(), x -> new ArrayList<>()).add(ref.getSourceRootUuid());
}

LinkedList<String> queue = new LinkedList<>();
Set<String> transitiveReferences = new HashSet<>();

// add all direct references
queue.addAll(rootConnections.getOrDefault(treeRootUuid, emptyList()));

// resolve all transitive references
while (!queue.isEmpty()) {
String uuid = queue.remove();
if (!transitiveReferences.contains(uuid)) {
queue.addAll(rootConnections.getOrDefault(treeRootUuid, emptyList()));
transitiveReferences.add(uuid);
}
}

return transitiveReferences;
}

}

+ 36
- 8
server/sonar-db-dao/src/main/resources/org/sonar/db/portfolio/PortfolioMapper.xml View File

@@ -16,6 +16,18 @@
p.updated_at as updatedAt
</sql>

<sql id="projectColumns">
p.uuid as uuid,
p.kee as kee,
p.qualifier as qualifier,
p.name as name,
p.description as description,
p.tags as tagsString,
p.private as isPrivate,
p.created_at as createdAt,
p.updated_at as updatedAt
</sql>

<select id="selectByUuid" parameterType="String" resultType="Portfolio">
SELECT
<include refid="portfolioColumns"/>
@@ -24,6 +36,17 @@
p.uuid=#{uuid,jdbcType=VARCHAR}
</select>

<select id="selectByUuids" parameterType="String" resultType="Portfolio">
SELECT
<include refid="portfolioColumns"/>
FROM portfolios p
where
p.uuid in
<foreach collection="uuids" open="(" close=")" item="uuid" separator=",">
#{uuid,jdbcType=VARCHAR}
</foreach>
</select>

<select id="selectByKey" parameterType="String" resultType="Portfolio">
SELECT
<include refid="portfolioColumns"/>
@@ -40,17 +63,22 @@
p.parent_uuid is null
</select>

<select id="selectProjects" resultType="String">
SELECT
p.project_uuid
FROM portfolio_projects p
where
p.portfolio_uuid=#{portfolioUuid,jdbcType=VARCHAR}
<select id="selectProjects" resultType="Project">
SELECT
<include refid="projectColumns"/>
FROM portfolio_projects pp
INNER JOIN projects p on p.uuid = pp.project_uuid
WHERE
pp.portfolio_uuid=#{portfolioUuid,jdbcType=VARCHAR}
</select>

<select id="selectAllProjectsInHierarchy" resultType="String">
<select id="selectAllProjectsInHierarchy" resultType="org.sonar.db.portfolio.PortfolioProjectDto">
SELECT
pp.project_uuid
pp.uuid,
pp.project_uuid as projectUuid,
pp.portfolio_uuid as portfolioUuid,
pp.project_uuid as projectUuid,
pp.created_at as createdAt
FROM portfolio_projects pp
INNER JOIN portfolios p
ON pp.portfolio_uuid = p.uuid

+ 1
- 1
server/sonar-db-dao/src/schema/schema-sq.ddl View File

@@ -580,7 +580,7 @@ ALTER TABLE "PORTFOLIO_REFERENCES" ADD CONSTRAINT "PK_PORTFOLIO_REFERENCES" PRIM

CREATE TABLE "PORTFOLIOS"(
"UUID" VARCHAR(40) NOT NULL,
"KEE" VARCHAR(40) NOT NULL,
"KEE" VARCHAR(400) NOT NULL,
"NAME" VARCHAR(2000) NOT NULL,
"DESCRIPTION" VARCHAR(2000),
"ROOT_UUID" VARCHAR(40) NOT NULL,

+ 18
- 8
server/sonar-db-dao/src/test/java/org/sonar/db/portfolio/PortfolioDaoTest.java View File

@@ -108,11 +108,10 @@ public class PortfolioDaoTest {
@Test
public void deleteByUuids() {
createPortfolio("p1");
createPortfolio("p2" );
createPortfolio("p2");
createPortfolio("p3");
createPortfolio("p4");


portfolioDao.addProject(db.getSession(), "p1", "proj1");
portfolioDao.addProject(db.getSession(), "p2", "proj1");

@@ -166,23 +165,29 @@ public class PortfolioDaoTest {

@Test
public void insert_and_select_projects() {
db.components().insertPrivateProject("project1");
db.components().insertPrivateProject("project2");

assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).isEmpty();
portfolioDao.addProject(db.getSession(), "portfolio1", "project1");
portfolioDao.addProject(db.getSession(), "portfolio1", "project2");
portfolioDao.addProject(db.getSession(), "portfolio2", "project2");
db.commit();
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).containsExactlyInAnyOrder("project1", "project2");
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).containsExactlyInAnyOrder("project2");
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).extracting(ProjectDto::getUuid).containsExactlyInAnyOrder("project1", "project2");
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).extracting(ProjectDto::getUuid).containsExactlyInAnyOrder("project2");
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio3")).isEmpty();

assertThat(db.countRowsOfTable("portfolio_projects")).isEqualTo(3);
assertThat(db.select(db.getSession(), "select created_at from portfolio_projects"))
.extracting(m -> m.values().iterator().next())
.containsExactlyInAnyOrder(1L, 2L, 3L);
.containsExactlyInAnyOrder(3L, 4L, 5L);
}

@Test
public void delete_projects() {
db.components().insertPrivateProject("project1");
db.components().insertPrivateProject("project2");

assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).isEmpty();
portfolioDao.addProject(db.getSession(), "portfolio1", "project1");
portfolioDao.addProject(db.getSession(), "portfolio1", "project2");
@@ -190,12 +195,17 @@ public class PortfolioDaoTest {

portfolioDao.deleteProjects(db.getSession(), "portfolio1");
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio1")).isEmpty();
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).containsExactlyInAnyOrder("project2");
assertThat(portfolioDao.getProjects(db.getSession(), "portfolio2")).extracting(ProjectDto::getUuid)
.containsExactlyInAnyOrder("project2");
}

@Test
public void getAllProjectsInHierarchy() {
db.components().insertPrivateProject("p1");
db.components().insertPrivateProject("p2");
db.components().insertPrivateProject("p3");
db.components().insertPrivateProject("p4");

createPortfolio("root", null, "root");
createPortfolio("child1", null, "root");
createPortfolio("child11", "child1", "root");
@@ -207,7 +217,7 @@ public class PortfolioDaoTest {
portfolioDao.addProject(db.getSession(), "child11", "p3");
portfolioDao.addProject(db.getSession(), "root2", "p4");

assertThat(portfolioDao.getAllProjectsInHierarchy(db.getSession(), "root")).containsOnly("p1", "p2", "p3");
assertThat(portfolioDao.getAllProjectsInHierarchy(db.getSession(), "root").keySet()).containsExactly("p1", "p2", "p3");
assertThat(portfolioDao.getAllProjectsInHierarchy(db.getSession(), "nonexisting")).isEmpty();
}


+ 61
- 6
server/sonar-db-dao/src/testFixtures/java/org/sonar/db/component/ComponentDbTester.java View File

@@ -27,12 +27,14 @@ import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
import org.sonar.db.portfolio.PortfolioDto;
import org.sonar.db.project.ProjectDto;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static java.util.Arrays.asList;
import static org.sonar.db.component.BranchType.BRANCH;
import static org.sonar.db.portfolio.PortfolioDto.SelectionMode.NONE;

public class ComponentDbTester {
private final DbTester db;
@@ -161,27 +163,67 @@ public class ComponentDbTester {
}

public final ComponentDto insertPublicPortfolio() {
return insertPublicPortfolio(defaults());
return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(false), false, defaults(), defaults());
}

public final ComponentDto insertPublicPortfolio(String uuid, Consumer<ComponentDto> dtoPopulator) {
return insertComponentImpl(ComponentTesting.newPortfolio(uuid).setPrivate(false), false, dtoPopulator);
return insertComponentAndPortfolio(ComponentTesting.newPortfolio(uuid).setPrivate(false), false, dtoPopulator, defaults());
}

public final ComponentDto insertPublicPortfolio(Consumer<ComponentDto> dtoPopulator) {
return insertComponentImpl(ComponentTesting.newPortfolio().setPrivate(false), false, dtoPopulator);
return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(false), false, dtoPopulator, defaults());
}

public final ComponentDto insertPublicPortfolio(Consumer<ComponentDto> dtoPopulator, Consumer<PortfolioDto> portfolioPopulator) {
return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(false), false, dtoPopulator, portfolioPopulator);
}

public ComponentDto insertComponentAndPortfolio(ComponentDto componentDto, boolean isPrivate, Consumer<ComponentDto> componentPopulator,
Consumer<PortfolioDto> portfolioPopulator) {
insertComponentImpl(componentDto, isPrivate, componentPopulator);

PortfolioDto portfolioDto = toPortfolioDto(componentDto, System2.INSTANCE.now());
portfolioPopulator.accept(portfolioDto);
dbClient.portfolioDao().insert(dbSession, portfolioDto);
db.commit();
return componentDto;
}

public final ComponentDto insertPrivatePortfolio() {
return insertComponentImpl(ComponentTesting.newPortfolio().setPrivate(true), true, defaults());
return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(true), true, defaults(), defaults());
}

public final ComponentDto insertPrivatePortfolio(String uuid, Consumer<ComponentDto> dtoPopulator) {
return insertComponentImpl(ComponentTesting.newPortfolio(uuid).setPrivate(true), true, dtoPopulator);
return insertComponentAndPortfolio(ComponentTesting.newPortfolio(uuid).setPrivate(true), true, dtoPopulator, defaults());
}

public final ComponentDto insertPrivatePortfolio(Consumer<ComponentDto> dtoPopulator) {
return insertComponentImpl(ComponentTesting.newPortfolio().setPrivate(true), true, dtoPopulator);
return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(true), true, dtoPopulator, defaults());
}

public final ComponentDto insertPrivatePortfolio(Consumer<ComponentDto> dtoPopulator, Consumer<PortfolioDto> portfolioPopulator) {
return insertComponentAndPortfolio(ComponentTesting.newPortfolio().setPrivate(true), true, dtoPopulator, portfolioPopulator);
}

public void addPortfolioProject(ComponentDto portfolio, String... projectUuids) {
for (String uuid : projectUuids) {
dbClient.portfolioDao().addProject(dbSession, portfolio.uuid(), uuid);
}
db.commit();
}

public void addPortfolioProject(ComponentDto portfolio, ComponentDto... projects) {
for (ComponentDto project : projects) {
dbClient.portfolioDao().addProject(dbSession, portfolio.uuid(), project.uuid());
}
db.commit();
}

public void addPortfolioProject(PortfolioDto portfolioDto, ProjectDto... projects) {
for (ProjectDto project : projects) {
dbClient.portfolioDao().addProject(dbSession, portfolioDto.getUuid(), project.getUuid());
}
db.commit();
}

public final ComponentDto insertPublicApplication() {
@@ -386,6 +428,19 @@ public class ComponentDbTester {
.setName(componentDto.name());
}

public static PortfolioDto toPortfolioDto(ComponentDto componentDto, long createTime) {
return new PortfolioDto()
.setUuid(componentDto.uuid())
.setKey(componentDto.getDbKey())
.setRootUuid(componentDto.projectUuid())
.setSelectionMode(NONE.name())
.setCreatedAt(createTime)
.setUpdatedAt(createTime)
.setPrivate(componentDto.isPrivate())
.setDescription(componentDto.description())
.setName(componentDto.name());
}

private static <T> Consumer<T> defaults() {
return t -> {
};

+ 1
- 1
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/CreatePortfoliosTable.java View File

@@ -40,7 +40,7 @@ public class CreatePortfoliosTable extends CreateTableChange {
public void execute(Context context, String tableName) throws SQLException {
context.execute(new CreateTableBuilder(getDialect(), tableName)
.addPkColumn(newVarcharColumnDefBuilder().setColumnName("uuid").setIsNullable(false).setLimit(UUID_SIZE).build())
.addColumn(newVarcharColumnDefBuilder().setColumnName("kee").setIsNullable(false).setLimit(UUID_SIZE).build())
.addColumn(newVarcharColumnDefBuilder().setColumnName("kee").setIsNullable(false).setLimit(400).build())
.addColumn(newVarcharColumnDefBuilder().setColumnName("name").setIsNullable(false).setLimit(2000).build())
.addColumn(newVarcharColumnDefBuilder().setColumnName("description").setIsNullable(true).setLimit(2000).build())
.addColumn(newVarcharColumnDefBuilder().setColumnName("root_uuid").setIsNullable(false).setLimit(UUID_SIZE).build())

+ 1
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91.java View File

@@ -39,6 +39,7 @@ public class DbVersion91 implements DbVersion {
.add(6011, "Create 'portfolios' table", CreatePortfoliosTable.class)
.add(6012, "Create 'portfolio_references' table", CreatePortfolioReferencesTable.class)
.add(6013, "Create 'portfolio_projects' table", CreatePortfolioProjectsTable.class)
.add(6014, "Migrate portfolios to new tables", MigratePortfoliosToNewTables.class)
;
}
}

+ 581
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTables.java View File

@@ -0,0 +1,581 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.version.v91;

import com.google.common.collect.Sets;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import javax.xml.XMLConstants;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.transform.sax.SAXSource;
import javax.xml.validation.SchemaFactory;
import org.apache.commons.lang.StringUtils;
import org.codehaus.staxmate.SMInputFactory;
import org.codehaus.staxmate.in.SMHierarchicCursor;
import org.codehaus.staxmate.in.SMInputCursor;
import org.sonar.api.utils.System2;
import org.sonar.core.util.UuidFactory;
import org.sonar.db.Database;
import org.sonar.db.DatabaseUtils;
import org.sonar.server.platform.db.migration.step.DataChange;
import org.sonar.server.platform.db.migration.step.MassUpdate;
import org.sonar.server.platform.db.migration.step.Select;
import org.sonar.server.platform.db.migration.step.Upsert;
import org.sonar.server.platform.db.migration.version.v91.MigratePortfoliosToNewTables.ViewXml.ViewDef;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.helpers.DefaultHandler;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Optional.ofNullable;
import static javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING;
import static org.apache.commons.io.IOUtils.toInputStream;
import static org.apache.commons.lang.StringUtils.trim;

public class MigratePortfoliosToNewTables extends DataChange {
private static final String SELECT_DEFAULT_VISIBILITY = "select text_value from properties where prop_key = 'projects.default.visibility'";
private static final String SELECT_UUID_VISIBILITY_BY_COMPONENT_KEY = "select c.uuid, c.private from components c where c.kee = ?";
private static final String SELECT_PORTFOLIO_UUID_AND_SELECTION_MODE_BY_KEY = "select uuid,selection_mode from portfolios where kee = ?";
private static final String SELECT_PROJECT_KEYS_BY_PORTFOLIO_UUID = "select p.kee from portfolio_projects pp "
+ "join projects p on pp.project_uuid = p.uuid where pp.portfolio_uuid = ?";
private static final String SELECT_PROJECT_UUIDS_BY_KEYS = "select p.uuid,p.kee from projects p where p.kee in (PLACEHOLDER)";
private static final String VIEWS_DEF_KEY = "views.def";
private static final String PLACEHOLDER = "PLACEHOLDER";

private final UuidFactory uuidFactory;
private final System2 system;

private boolean defaultPrivateFlag;

public enum SelectionMode {
NONE, MANUAL, REGEXP, REST, TAGS
}

public MigratePortfoliosToNewTables(Database db, UuidFactory uuidFactory, System2 system) {
super(db);

this.uuidFactory = uuidFactory;
this.system = system;
}

@Override
protected void execute(Context context) throws SQLException {
String xml = getViewsDefinition(context);
// skip migration if `views.def` does not exist in the db
if (xml == null) {
return;
}

this.defaultPrivateFlag = ofNullable(context.prepareSelect(SELECT_DEFAULT_VISIBILITY)
.get(row -> "private".equals(row.getString(1))))
.orElse(false);

try {
Map<String, ViewXml.ViewDef> portfolioXmlMap = ViewXml.parse(xml);
List<ViewXml.ViewDef> portfolios = new LinkedList<>(portfolioXmlMap.values());

Map<String, PortfolioDb> portfolioDbMap = new HashMap<>();
for (ViewXml.ViewDef portfolio : portfolios) {
PortfolioDb createdPortfolio = insertPortfolio(context, portfolio);
if (createdPortfolio.selectionMode == SelectionMode.MANUAL) {
insertPortfolioProjects(context, portfolio, createdPortfolio);
}
portfolioDbMap.put(createdPortfolio.kee, createdPortfolio);
}
// all portfolio has been created and new uuids assigned
// update portfolio hierarchy parent/root
insertReferences(context, portfolioXmlMap, portfolioDbMap);
updateHierarchy(context, portfolioXmlMap, portfolioDbMap);
} catch (Exception e) {
throw new IllegalStateException("Failed to migrate views definitions property.", e);
}
}

private PortfolioDb insertPortfolio(Context context, ViewXml.ViewDef portfolioFromXml) throws SQLException {
long now = system.now();
PortfolioDb portfolioDb = context.prepareSelect(SELECT_PORTFOLIO_UUID_AND_SELECTION_MODE_BY_KEY)
.setString(1, portfolioFromXml.key)
.get(r -> new PortfolioDb(r.getString(1), portfolioFromXml.key, SelectionMode.valueOf(r.getString(2))));

Optional<ComponentDb> componentDbOpt = ofNullable(context.prepareSelect(SELECT_UUID_VISIBILITY_BY_COMPONENT_KEY)
.setString(1, portfolioFromXml.key)
.get(row -> new ComponentDb(row.getString(1), row.getBoolean(2))));

// no portfolio -> insert
if (portfolioDb == null) {
Upsert insertPortfolioQuery = context.prepareUpsert("insert into " +
"portfolios(uuid, kee, private, name, description, root_uuid, parent_uuid, selection_mode, selection_expression, updated_at, created_at) " +
"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");

String portfolioUuid = componentDbOpt.map(c -> c.uuid).orElse(uuidFactory.create());
insertPortfolioQuery.setString(1, portfolioUuid)
.setString(2, portfolioFromXml.key)
.setBoolean(3, componentDbOpt.map(c -> c.visibility).orElse(this.defaultPrivateFlag))
.setString(4, portfolioFromXml.name)
.setString(5, portfolioFromXml.desc)
.setString(6, PLACEHOLDER)
.setString(7, PLACEHOLDER);
SelectionMode selectionMode = SelectionMode.NONE;
String selectionExpression = null;
if (portfolioFromXml.getProjects() != null && !portfolioFromXml.getProjects().isEmpty()) {
selectionMode = SelectionMode.MANUAL;
} else if (portfolioFromXml.regexp != null && !portfolioFromXml.regexp.isBlank()) {
selectionMode = SelectionMode.REGEXP;
selectionExpression = portfolioFromXml.regexp;
} else if (portfolioFromXml.def) {
selectionMode = SelectionMode.REST;
} else if (portfolioFromXml.tagsAssociation != null && !portfolioFromXml.tagsAssociation.isEmpty()) {
selectionMode = SelectionMode.TAGS;
selectionExpression = String.join(",", portfolioFromXml.tagsAssociation);
}

insertPortfolioQuery.setString(8, selectionMode.name())
.setString(9, selectionExpression)
// set dates
.setLong(10, now)
.setLong(11, now)
.execute()
.commit();
return new PortfolioDb(portfolioUuid, portfolioFromXml.key, selectionMode);
}
return portfolioDb;
}

private void insertPortfolioProjects(Context context, ViewDef portfolio, PortfolioDb createdPortfolio) throws SQLException {
long now = system.now();
// select all already added project uuids
Set<String> alreadyAddedPortfolioProjects = new HashSet<>(
context.prepareSelect(SELECT_PROJECT_KEYS_BY_PORTFOLIO_UUID)
.setString(1, createdPortfolio.uuid)
.list(r -> r.getString(1)));

Set<String> projectKeysFromXml = new HashSet<>(portfolio.getProjects());
Set<String> projectKeysToBeAdded = Sets.difference(projectKeysFromXml, alreadyAddedPortfolioProjects);

if (!projectKeysToBeAdded.isEmpty()) {
List<ProjectDb> projects = findPortfolioProjects(context, projectKeysToBeAdded);

var upsert = context.prepareUpsert("insert into " +
"portfolio_projects(uuid, portfolio_uuid, project_uuid, created_at) " +
"values (?, ?, ?, ?)");
for (ProjectDb projectDb : projects) {
upsert.setString(1, uuidFactory.create())
.setString(2, createdPortfolio.uuid)
.setString(3, projectDb.uuid)
.setLong(4, now)
.addBatch();
}
if (!projects.isEmpty()) {
upsert.execute()
.commit();
}
}
}

private static List<ProjectDb> findPortfolioProjects(Context context, Set<String> projectKeysToBeAdded) {
return DatabaseUtils.executeLargeInputs(projectKeysToBeAdded, keys -> {
try {
String selectQuery = SELECT_PROJECT_UUIDS_BY_KEYS.replace(PLACEHOLDER,
keys.stream().map(key -> "'" + key + "'").collect(
Collectors.joining(",")));
return context.prepareSelect(selectQuery)
.list(r -> new ProjectDb(r.getString(1), r.getString(2)));
} catch (SQLException e) {
throw new IllegalStateException("Could not execute 'in' query", e);
}
});
}

private void insertReferences(Context context, Map<String, ViewDef> portfolioXmlMap,
Map<String, PortfolioDb> portfolioDbMap) throws SQLException {
Upsert insertQuery = context.prepareUpsert("insert into portfolio_references(uuid, portfolio_uuid, reference_uuid, created_at) values (?, ?, ?, ?)");

long now = system.now();
boolean shouldExecuteQuery = false;
for (ViewDef portfolio : portfolioXmlMap.values()) {
var currentPortfolioUuid = portfolioDbMap.get(portfolio.key).uuid;
Set<String> referencesFromXml = new HashSet<>(portfolio.getReferences());
Set<String> referencesFromDb = new HashSet<>(context.prepareSelect("select pr.reference_uuid from portfolio_references pr where pr.portfolio_uuid = ?")
.setString(1, currentPortfolioUuid)
.list(row -> row.getString(1)));

for (String appOrPortfolio : referencesFromXml) {
// if portfolio and hasn't been added already
if (portfolioDbMap.containsKey(appOrPortfolio) && !referencesFromDb.contains(portfolioDbMap.get(appOrPortfolio).uuid)) {
insertQuery
.setString(1, uuidFactory.create())
.setString(2, currentPortfolioUuid)
.setString(3, portfolioDbMap.get(appOrPortfolio).uuid)
.setLong(4, now)
.addBatch();
shouldExecuteQuery = true;
} else {
// if application exist and haven't been added
String appUuid = context.prepareSelect("select p.uuid from projects p where p.kee = ?")
.setString(1, appOrPortfolio)
.get(row -> row.getString(1));
if (appUuid != null && !referencesFromDb.contains(appUuid)) {
insertQuery
.setString(1, uuidFactory.create())
.setString(2, currentPortfolioUuid)
.setString(3, appUuid)
.setLong(4, now)
.addBatch();
shouldExecuteQuery = true;
}
}
}
}
if (shouldExecuteQuery) {
insertQuery
.execute()
.commit();
}

}

private static void updateHierarchy(Context context, Map<String, ViewXml.ViewDef> defs, Map<String, PortfolioDb> portfolioDbMap) throws SQLException {
MassUpdate massUpdate = context.prepareMassUpdate();
massUpdate.select("select uuid, kee from portfolios where root_uuid = ? or parent_uuid = ?")
.setString(1, PLACEHOLDER)
.setString(2, PLACEHOLDER);
massUpdate.update("update portfolios set root_uuid = ?, parent_uuid = ? where uuid = ?");
massUpdate.execute((row, update) -> {
String currentPortfolioUuid = row.getString(1);
String currentPortfolioKey = row.getString(2);

var currentPortfolio = defs.get(currentPortfolioKey);
String parentUuid = ofNullable(currentPortfolio.parent).map(parent -> portfolioDbMap.get(parent).uuid).orElse(null);
String rootUuid = ofNullable(currentPortfolio.root).map(root -> portfolioDbMap.get(root).uuid).orElse(currentPortfolioUuid);
update.setString(1, rootUuid)
.setString(2, parentUuid)
.setString(3, currentPortfolioUuid);
return true;
});
}

@CheckForNull
private static String getViewsDefinition(DataChange.Context context) throws SQLException {
Select select = context.prepareSelect("select text_value,clob_value from internal_properties where kee=?");
select.setString(1, VIEWS_DEF_KEY);
return select.get(row -> {
String v = row.getString(1);
if (v != null) {
return v;
} else {
return row.getString(2);
}
});
}

private static class ComponentDb {
String uuid;
boolean visibility;

public ComponentDb(String uuid, boolean visibility) {
this.uuid = uuid;
this.visibility = visibility;
}
}

private static class PortfolioDb {
String uuid;
String kee;
SelectionMode selectionMode;

PortfolioDb(String uuid, String kee,
SelectionMode selectionMode) {
this.uuid = uuid;
this.kee = kee;
this.selectionMode = selectionMode;
}
}

private static class ProjectDb {
String uuid;
String kee;

ProjectDb(String uuid, String kee) {
this.uuid = uuid;
this.kee = kee;
}
}

static class ViewXml {
static final String SCHEMA_VIEWS = "/static/views.xsd";
static final String VIEWS_HEADER_BARE = "<views>";
static final Pattern VIEWS_HEADER_BARE_PATTERN = Pattern.compile(VIEWS_HEADER_BARE);
static final String VIEWS_HEADER_FQ = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ "<views xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns=\"http://sonarsource.com/schema/views\">";

private ViewXml() {
// nothing to do here
}

private static Map<String, ViewDef> parse(String xml) throws ParserConfigurationException, SAXException, IOException, XMLStreamException {
if (StringUtils.isEmpty(xml)) {
return new LinkedHashMap<>(0);
}

List<ViewDef> views;
validate(xml);
SMInputFactory inputFactory = initStax();
SMHierarchicCursor rootC = inputFactory.rootElementCursor(new StringReader(xml));
rootC.advance(); // <views>
SMInputCursor cursor = rootC.childElementCursor();
views = parseViewDefinitions(cursor);

Map<String, ViewDef> result = new LinkedHashMap<>(views.size());
for (ViewDef def : views) {
result.put(def.key, def);
}

return result;
}

private static void validate(String xml) throws IOException, SAXException, ParserConfigurationException {
// Replace bare, namespace unaware header with fully qualified header (with schema declaration)
String fullyQualifiedXml = VIEWS_HEADER_BARE_PATTERN.matcher(xml).replaceFirst(VIEWS_HEADER_FQ);
try (InputStream xsd = MigratePortfoliosToNewTables.class.getResourceAsStream(SCHEMA_VIEWS)) {
InputSource viewsDefinition = new InputSource(new InputStreamReader(toInputStream(fullyQualifiedXml, UTF_8), UTF_8));

SchemaFactory saxSchemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");

SAXParserFactory parserFactory = SAXParserFactory.newInstance();
parserFactory.setFeature(FEATURE_SECURE_PROCESSING, true);
parserFactory.setNamespaceAware(true);
parserFactory.setSchema(saxSchemaFactory.newSchema(new SAXSource(new InputSource(xsd))));

SAXParser saxParser = parserFactory.newSAXParser();
saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
saxParser.parse(viewsDefinition, new ViewsValidator());
}
}

private static List<ViewDef> parseViewDefinitions(SMInputCursor viewsCursor) throws XMLStreamException {
List<ViewDef> views = new ArrayList<>();
while (viewsCursor.getNext() != null) {
ViewDef viewDef = new ViewDef();
viewDef.setKey(viewsCursor.getAttrValue("key"));
viewDef.setDef(Boolean.parseBoolean(viewsCursor.getAttrValue("def")));
viewDef.setParent(viewsCursor.getAttrValue("parent"));
viewDef.setRoot(viewsCursor.getAttrValue("root"));
SMInputCursor viewCursor = viewsCursor.childElementCursor();
while (viewCursor.getNext() != null) {
String nodeName = viewCursor.getLocalName();
parseChildElement(viewDef, viewCursor, nodeName);
}
views.add(viewDef);
}
return views;
}

private static void parseChildElement(ViewDef viewDef, SMInputCursor viewCursor, String nodeName) throws XMLStreamException {
if (StringUtils.equals(nodeName, "name")) {
viewDef.setName(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "desc")) {
viewDef.setDesc(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "regexp")) {
viewDef.setRegexp(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "language")) {
viewDef.setLanguage(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "tag_key")) {
viewDef.setTagKey(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "tag_value")) {
viewDef.setTagValue(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "p")) {
viewDef.addProject(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "vw-ref")) {
viewDef.addReference(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "qualifier")) {
viewDef.setQualifier(trim(viewCursor.collectDescendantText()));
} else if (StringUtils.equals(nodeName, "tagsAssociation")) {
parseTagsAssociation(viewDef, viewCursor);
}
}

private static void parseTagsAssociation(ViewDef def, SMInputCursor viewCursor) throws XMLStreamException {
SMInputCursor projectCursor = viewCursor.childElementCursor();
while (projectCursor.getNext() != null) {
def.addTagAssociation(trim(projectCursor.collectDescendantText()));
}
}

private static SMInputFactory initStax() {
XMLInputFactory xmlFactory = XMLInputFactory.newInstance();
xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE);
xmlFactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, Boolean.FALSE);
// just so it won't try to load DTD in if there's DOCTYPE
xmlFactory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
xmlFactory.setProperty(XMLInputFactory.IS_VALIDATING, Boolean.FALSE);
return new SMInputFactory(xmlFactory);
}

static class ViewDef {
String key = null;

String parent = null;

String root = null;

boolean def = false;

List<String> p = new ArrayList<>();

List<String> vwRef = new ArrayList<>();

String name = null;

String desc = null;

String regexp = null;

String language = null;

String tagKey = null;

String tagValue = null;

String qualifier = null;

Set<String> tagsAssociation = new TreeSet<>();

public List<String> getProjects() {
return p;
}

public List<String> getReferences() {
return vwRef;
}

public ViewDef setKey(String key) {
this.key = key;
return this;
}

public ViewDef setParent(String parent) {
this.parent = parent;
return this;
}

public ViewDef setRoot(@Nullable String root) {
this.root = root;
return this;
}

public ViewDef setDef(boolean def) {
this.def = def;
return this;
}

public ViewDef addProject(String project) {
this.p.add(project);
return this;
}

public ViewDef setName(String name) {
this.name = name;
return this;
}

public ViewDef setDesc(@Nullable String desc) {
this.desc = desc;
return this;
}

public ViewDef setRegexp(@Nullable String regexp) {
this.regexp = regexp;
return this;
}

public ViewDef setLanguage(@Nullable String language) {
this.language = language;
return this;
}

public ViewDef setTagKey(@Nullable String tagKey) {
this.tagKey = tagKey;
return this;
}

public ViewDef setTagValue(@Nullable String tagValue) {
this.tagValue = tagValue;
return this;
}

public ViewDef addReference(String reference) {
this.vwRef.add(reference);
return this;
}

public ViewDef setQualifier(@Nullable String qualifier) {
this.qualifier = qualifier;
return this;
}

public ViewDef addTagAssociation(String tag) {
this.tagsAssociation.add(tag);
return this;
}
}

private static final class ViewsValidator extends DefaultHandler {
@Override
public void error(SAXParseException exception) throws SAXException {
throw exception;
}

@Override
public void warning(SAXParseException exception) throws SAXException {
throw exception;
}

@Override
public void fatalError(SAXParseException exception) throws SAXException {
throw exception;
}
}
}
}

+ 1
- 1
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/DbVersion91Test.java View File

@@ -41,7 +41,7 @@ public class DbVersion91Test {

@Test
public void verify_migration_count() {
verifyMigrationCount(underTest, 13);
verifyMigrationCount(underTest, 14);
}

}

+ 431
- 0
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest.java View File

@@ -0,0 +1,431 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.version.v91;

import java.sql.SQLException;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.impl.utils.TestSystem2;
import org.sonar.api.resources.Qualifiers;
import org.sonar.api.utils.System2;
import org.sonar.core.util.UuidFactory;
import org.sonar.core.util.UuidFactoryFast;
import org.sonar.db.CoreDbTester;
import org.sonar.server.platform.db.migration.step.DataChange;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.tuple;

public class MigratePortfoliosToNewTablesTest {
static final int TEXT_VALUE_MAX_LENGTH = 4000;
private static final long NOW = 10_000_000L;

@Rule
public final CoreDbTester db = CoreDbTester.createForSchema(MigratePortfoliosToNewTablesTest.class, "schema.sql");

private final UuidFactory uuidFactory = UuidFactoryFast.getInstance();
private final System2 system2 = new TestSystem2().setNow(NOW);

private final DataChange underTest = new MigratePortfoliosToNewTables(db.database(), uuidFactory, system2);

private final String SIMPLE_XML_ONE_PORTFOLIO = "<views>" +
"<vw key=\"port1\" def=\"false\">\n" +
" <name><![CDATA[name]]></name>\n" +
" <desc><![CDATA[description]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" </vw>\n" +
"<vw key=\"port2\" def=\"false\">\n" +
" <name><![CDATA[name2]]></name>\n" +
" <desc><![CDATA[description2]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" </vw>\n" +
"</views>";

private final String SIMPLE_XML_ONLY_PORTFOLIOS = "<views>" +
"<vw key=\"port1\" def=\"true\">\n" +
" <name><![CDATA[port1]]></name>\n" +
" <desc><![CDATA[]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" </vw>\n" +
"<vw key=\"port3\" def=\"false\">\n" +
" <name><![CDATA[port3]]></name>\n" +
" <desc><![CDATA[]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" </vw>\n" +
"<vw key=\"port2\" def=\"false\">\n" +
" <name><![CDATA[port2]]></name>\n" +
" <desc><![CDATA[]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" <vw-ref><![CDATA[port1]]></vw-ref>\n" +
" <vw-ref><![CDATA[port3]]></vw-ref>\n" +
" </vw>\n" +
"</views>";

private final String SIMPLE_XML_PORTFOLIOS_AND_APP = "<views>" +
"<vw key=\"port1\" def=\"true\">\n" +
" <name><![CDATA[port1]]></name>\n" +
" <desc><![CDATA[]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" </vw>\n" +
"<vw key=\"port3\" def=\"false\">\n" +
" <name><![CDATA[port3]]></name>\n" +
" <desc><![CDATA[]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" </vw>\n" +
"<vw key=\"port2\" def=\"false\">\n" +
" <name><![CDATA[port2]]></name>\n" +
" <desc><![CDATA[]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" <vw-ref><![CDATA[port1]]></vw-ref>\n" +
" <vw-ref><![CDATA[port3]]></vw-ref>\n" +
" </vw>\n" +
"<vw key=\"port4\" def=\"false\">\n" +
" <name><![CDATA[port2]]></name>\n" +
" <desc><![CDATA[]]></desc>\n" +
" <qualifier><![CDATA[VW]]></qualifier>\n" +
" <vw-ref><![CDATA[port2]]></vw-ref>\n" +
" <vw-ref><![CDATA[app1-key]]></vw-ref>\n" +
" <vw-ref><![CDATA[app2-key]]></vw-ref>\n" +
" <vw-ref><![CDATA[app3-key-not-existing]]></vw-ref>\n" +
" </vw>\n" +
"</views>";

private final String SIMPLE_XML_PORTFOLIOS_HIERARCHY = "<views>" +
" <vw key=\"port1\" def=\"false\">\n"
+ " <name><![CDATA[port1]]></name>\n"
+ " <desc><![CDATA[port1]]></desc>\n"
+ " <qualifier><![CDATA[VW]]></qualifier>\n"
+ " </vw>\n"
+ " <vw key=\"port2\" def=\"false\" root=\"port1\" parent=\"port1\">\n"
+ " <name><![CDATA[port2]]></name>\n"
+ " <desc><![CDATA[port2]]></desc>\n"
+ " </vw>\n"
+ " <vw key=\"port3\" def=\"false\" root=\"port1\" parent=\"port1\">\n"
+ " <name><![CDATA[port3]]></name>\n"
+ " <desc><![CDATA[port3]]></desc>\n"
+ " </vw>\n"
+ " <vw key=\"port4\" def=\"false\" root=\"port1\" parent=\"port1\">\n"
+ " <name><![CDATA[port4]]></name>\n"
+ " <desc><![CDATA[port4]]></desc>\n"
+ " </vw>\n"
+ " <vw key=\"port5\" def=\"false\" root=\"port1\" parent=\"port2\">\n"
+ " <name><![CDATA[port5]]></name>\n"
+ " </vw>\n"
+ " <vw key=\"port6\" def=\"false\" root=\"port1\" parent=\"port3\">\n"
+ " <name><![CDATA[port6]]></name>\n"
+ " <desc><![CDATA[port6]]></desc>\n"
+ " </vw>\n"
+ " <vw key=\"port7\" def=\"false\" root=\"port1\" parent=\"port3\">\n"
+ " <name><![CDATA[port7]]></name>\n"
+ " <desc><![CDATA[port7]]></desc>\n"
+ " </vw>" +
"</views>";

private final String SIMPLE_XML_PORTFOLIOS_DIFFERENT_SELECTIONS = "<views>"
+ " <vw key=\"port-regexp\" def=\"false\">\n"
+ " <name><![CDATA[port-regexp]]></name>\n"
+ " <desc><![CDATA[port-regexp]]></desc>\n"
+ " <regexp><![CDATA[.*port.*]]></regexp>\n"
+ " <qualifier><![CDATA[VW]]></qualifier>\n"
+ " </vw>"
+ " <vw key=\"port-tags\" def=\"false\">\n"
+ " <name><![CDATA[port-tags]]></name>\n"
+ " <desc><![CDATA[port-tags]]></desc>\n"
+ " <qualifier><![CDATA[VW]]></qualifier>\n"
+ " <tagsAssociation>\n"
+ " <tag>tag1</tag>\n"
+ " <tag>tag2</tag>\n"
+ " <tag>tag3</tag>\n"
+ " </tagsAssociation>\n"
+ " </vw>"
+ " <vw key=\"port-projects\" def=\"false\">\n"
+ " <name><![CDATA[port-projects]]></name>\n"
+ " <desc><![CDATA[port-projects]]></desc>\n"
+ " <qualifier><![CDATA[VW]]></qualifier>\n"
+ " <p>p1-key</p>\n"
+ " <p>p2-key</p>\n"
+ " <p>p3-key</p>\n"
+ " <p>p4-key-not-existing</p>\n"
+ " </vw>"
+ "</views>";

private final String XML_PORTFOLIOS_LEGACY_SELECTIONS = "<views>"
+ " <vw key=\"port-language\" def=\"false\">\n"
+ " <name><![CDATA[port-language]]></name>\n"
+ " <desc><![CDATA[port-language]]></desc>\n"
+ " <language><![CDATA[javascript]]></language>\n"
+ " <qualifier><![CDATA[VW]]></qualifier>\n"
+ " </vw>"
+ " <vw key=\"port-tag-value\" def=\"false\">\n"
+ " <name><![CDATA[port-tag-value]]></name>\n"
+ " <desc><![CDATA[port-tag-value]]></desc>\n"
+ " <qualifier><![CDATA[VW]]></qualifier>\n"
+ " <tag_key>tag-key</tag_key>\n"
+ " <tag_value>tag-value</tag_value>\n"
+ " </vw>"
+ "</views>";

@Test
public void does_not_fail_when_nothing_to_migrate() throws SQLException {
assertThatCode(underTest::execute)
.doesNotThrowAnyException();
}

@Test
public void migrate_single_portfolio_with_default_visibility() throws SQLException {
insertViewsDefInternalProperty(SIMPLE_XML_ONE_PORTFOLIO);
insertDefaultVisibilityProperty(true);
insertComponent("uuid", "port2", false);

underTest.execute();
// re-entrant
underTest.execute();

assertThat(db.select("select kee, private, name, description, "
+ "selection_mode, selection_expression, "
+ "updated_at, created_at from portfolios"))
.extracting(
row -> row.get("KEE"),
row -> row.get("PRIVATE"),
row -> row.get("NAME"),
row -> row.get("DESCRIPTION"),
row -> row.get("SELECTION_MODE"),
row -> row.get("SELECTION_EXPRESSION"),
row -> row.get("UPDATED_AT"),
row -> row.get("CREATED_AT"))
.containsExactlyInAnyOrder(
tuple("port1", true, "name", "description", "NONE", null, NOW, NOW),
tuple("port2", false, "name2", "description2", "NONE", null, NOW, NOW));
}

@Test
public void migrate_portfolios_should_assign_component_uuid() throws SQLException {
insertViewsDefInternalProperty(SIMPLE_XML_ONE_PORTFOLIO);
insertComponent("uuid1", "port1", true);
insertComponent("uuid2", "port2", false);

underTest.execute();
// re-entrant
underTest.execute();

assertThat(db.select("select uuid, kee, private, name, description, "
+ "selection_mode, selection_expression, "
+ "updated_at, created_at from portfolios"))
.extracting(
row -> row.get("UUID"),
row -> row.get("KEE"),
row -> row.get("PRIVATE"),
row -> row.get("NAME"),
row -> row.get("DESCRIPTION"),
row -> row.get("SELECTION_MODE"),
row -> row.get("SELECTION_EXPRESSION"),
row -> row.get("UPDATED_AT"),
row -> row.get("CREATED_AT"))
.containsExactlyInAnyOrder(
tuple("uuid1", "port1", true, "name", "description", "NONE", null, NOW, NOW),
tuple("uuid2", "port2", false, "name2", "description2", "NONE", null, NOW, NOW));
}

@Test
public void migrate_simple_xml_only_portfolios_references() throws SQLException {
insertViewsDefInternalProperty(SIMPLE_XML_ONLY_PORTFOLIOS);

underTest.execute();
// re-entrant
underTest.execute();

assertThat(db.select("select kee from portfolios"))
.extracting(row -> row.get("KEE"))
.containsExactlyInAnyOrder("port1", "port2", "port3");

var portfolioKeyByUuid = db.select("select uuid, kee from portfolios").stream()
.collect(Collectors.toMap(row -> row.get("KEE"), row -> row.get("UUID").toString()));

String portfolio1Uuid = portfolioKeyByUuid.get("port1");
String portfolio2Uuid = portfolioKeyByUuid.get("port2");
String portfolio3Uuid = portfolioKeyByUuid.get("port3");

assertThat(db.select("select portfolio_uuid, reference_uuid from portfolio_references"))
.extracting(row -> row.get("PORTFOLIO_UUID"), row -> row.get("REFERENCE_UUID"))
.containsExactlyInAnyOrder(tuple(portfolio2Uuid, portfolio1Uuid), tuple(portfolio2Uuid, portfolio3Uuid));
}

@Test
public void migrate_simple_xml_portfolios_and_apps_references() throws SQLException {
insertViewsDefInternalProperty(SIMPLE_XML_PORTFOLIOS_AND_APP);

insertProject("app1-uuid", "app1-key", Qualifiers.APP);
insertProject("app2-uuid", "app2-key", Qualifiers.APP);
insertProject("proj1", "proj1-key", Qualifiers.PROJECT);

underTest.execute();
// re-entrant
underTest.execute();

assertThat(db.select("select kee from portfolios"))
.extracting(row -> row.get("KEE"))
.containsExactlyInAnyOrder("port1", "port2", "port3", "port4");

var portfolioKeyByUuid = db.select("select uuid, kee from portfolios").stream()
.collect(Collectors.toMap(row -> row.get("KEE"), row -> row.get("UUID").toString()));

String portfolio1Uuid = portfolioKeyByUuid.get("port1");
String portfolio2Uuid = portfolioKeyByUuid.get("port2");
String portfolio3Uuid = portfolioKeyByUuid.get("port3");
String portfolio4Uuid = portfolioKeyByUuid.get("port4");

assertThat(db.select("select portfolio_uuid, reference_uuid from portfolio_references"))
.extracting(row -> row.get("PORTFOLIO_UUID"), row -> row.get("REFERENCE_UUID"))
.containsExactlyInAnyOrder(
tuple(portfolio2Uuid, portfolio1Uuid),
tuple(portfolio2Uuid, portfolio3Uuid),
tuple(portfolio4Uuid, portfolio2Uuid),
tuple(portfolio4Uuid, "app1-uuid"),
tuple(portfolio4Uuid, "app2-uuid"))
.doesNotContain(tuple(portfolio4Uuid, "app3-key-not-existing"));
}

@Test
public void migrate_xml_portfolios_hierarchy() throws SQLException {
insertViewsDefInternalProperty(SIMPLE_XML_PORTFOLIOS_HIERARCHY);

underTest.execute();
// re-entrant
underTest.execute();

assertThat(db.select("select kee from portfolios"))
.extracting(row -> row.get("KEE"))
.containsExactlyInAnyOrder("port1", "port2", "port3", "port4", "port5", "port6", "port7");

var portfolioKeyByUuid = db.select("select uuid, kee from portfolios").stream()
.collect(Collectors.toMap(row -> row.get("KEE"), row -> row.get("UUID").toString()));

String portfolio1Uuid = portfolioKeyByUuid.get("port1");
String portfolio2Uuid = portfolioKeyByUuid.get("port2");
String portfolio3Uuid = portfolioKeyByUuid.get("port3");
String portfolio4Uuid = portfolioKeyByUuid.get("port4");
String portfolio5Uuid = portfolioKeyByUuid.get("port5");
String portfolio6Uuid = portfolioKeyByUuid.get("port6");
String portfolio7Uuid = portfolioKeyByUuid.get("port7");

assertThat(db.select("select uuid, parent_uuid, root_uuid from portfolios"))
.extracting(row -> row.get("UUID"), row -> row.get("PARENT_UUID"), row -> row.get("ROOT_UUID"))
.containsExactlyInAnyOrder(
tuple(portfolio1Uuid, null, portfolio1Uuid),
tuple(portfolio2Uuid, portfolio1Uuid, portfolio1Uuid),
tuple(portfolio3Uuid, portfolio1Uuid, portfolio1Uuid),
tuple(portfolio4Uuid, portfolio1Uuid, portfolio1Uuid),
tuple(portfolio5Uuid, portfolio2Uuid, portfolio1Uuid),
tuple(portfolio6Uuid, portfolio3Uuid, portfolio1Uuid),
tuple(portfolio7Uuid, portfolio3Uuid, portfolio1Uuid));
}

@Test
public void migrate_xml_portfolios_different_selections() throws SQLException {
insertViewsDefInternalProperty(SIMPLE_XML_PORTFOLIOS_DIFFERENT_SELECTIONS);

insertProject("p1-uuid", "p1-key", Qualifiers.PROJECT);
insertProject("p2-uuid", "p2-key", Qualifiers.PROJECT);
insertProject("p3-uuid", "p3-key", Qualifiers.PROJECT);

underTest.execute();
// re-entrant
underTest.execute();

assertThat(db.select("select kee, selection_mode, selection_expression from portfolios"))
.extracting(row -> row.get("KEE"), row -> row.get("SELECTION_MODE"), row -> row.get("SELECTION_EXPRESSION"))
.containsExactlyInAnyOrder(
tuple("port-regexp", "REGEXP", ".*port.*"),
tuple("port-projects", "MANUAL", null),
tuple("port-tags", "TAGS", "tag1,tag2,tag3"));

// verify projects
assertThat(db.select("select p.kee, pp.project_uuid from "
+ "portfolio_projects pp join portfolios p on pp.portfolio_uuid = p.uuid where p.kee = 'port-projects'"))
.extracting(row -> row.get("KEE"), row -> row.get("PROJECT_UUID"))
.containsExactlyInAnyOrder(
tuple("port-projects", "p1-uuid"),
tuple("port-projects", "p2-uuid"),
tuple("port-projects", "p3-uuid"))
.doesNotContain(tuple("port-projects", "p4-uuid-not-existing"));
}

@Test
public void migrate_xml_portfolios_legacy_selections() throws SQLException {
insertViewsDefInternalProperty(XML_PORTFOLIOS_LEGACY_SELECTIONS);
underTest.execute();
// re-entrant
underTest.execute();

assertThat(db.select("select kee, selection_mode, selection_expression from portfolios"))
.extracting(row -> row.get("KEE"), row -> row.get("SELECTION_MODE"), row -> row.get("SELECTION_EXPRESSION"))
.containsExactlyInAnyOrder(
tuple("port-language", "NONE", null),
tuple("port-tag-value", "NONE", null));
}

private void insertProject(String uuid, String key, String qualifier) {
db.executeInsert("PROJECTS",
"UUID", uuid,
"KEE", key,
"QUALIFIER", qualifier,
"ORGANIZATION_UUID", uuid + "-key",
"TAGS", "tag1",
"PRIVATE", Boolean.toString(false),
"UPDATED_AT", System2.INSTANCE.now());
}

private void insertViewsDefInternalProperty(@Nullable String xml) {
String valueColumn = "text_value";
if (xml != null && xml.length() > TEXT_VALUE_MAX_LENGTH) {
valueColumn = "clob_value";
}

db.executeInsert("internal_properties",
"kee", "views.def",
"is_empty", "false",
valueColumn, xml,
"created_at", system2.now());
}

private void insertComponent(String uuid, String kee, boolean isPrivate) {
db.executeInsert("components",
"uuid", uuid,
"kee", kee,
"enabled", false,
"private", isPrivate,
"root_uuid", uuid,
"uuid_path", uuid,
"project_uuid", uuid);
}

private void insertDefaultVisibilityProperty(boolean isPrivate) {
db.executeInsert("properties",
"uuid", uuidFactory.create(),
"prop_key", "projects.default.visibility",
"IS_EMPTY", false,
"text_value", isPrivate ? "private" : "public",
"created_at", system2.now());
}

}

+ 110
- 0
server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v91/MigratePortfoliosToNewTablesTest/schema.sql View File

@@ -0,0 +1,110 @@
CREATE TABLE "PROJECTS"(
"UUID" VARCHAR(40) NOT NULL,
"KEE" VARCHAR(400) NOT NULL,
"QUALIFIER" VARCHAR(10) NOT NULL,
"ORGANIZATION_UUID" VARCHAR(40) NOT NULL,
"NAME" VARCHAR(2000),
"DESCRIPTION" VARCHAR(2000),
"PRIVATE" BOOLEAN NOT NULL,
"TAGS" VARCHAR(500),
"CREATED_AT" BIGINT,
"UPDATED_AT" BIGINT NOT NULL
);
ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID");
CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE");
CREATE INDEX "IDX_QUALIFIER" ON "PROJECTS"("QUALIFIER");

CREATE TABLE "INTERNAL_PROPERTIES"(
"KEE" VARCHAR(20) NOT NULL,
"IS_EMPTY" BOOLEAN NOT NULL,
"TEXT_VALUE" VARCHAR(4000),
"CLOB_VALUE" CLOB,
"CREATED_AT" BIGINT NOT NULL
);
ALTER TABLE "INTERNAL_PROPERTIES" ADD CONSTRAINT "PK_INTERNAL_PROPERTIES" PRIMARY KEY("KEE");

CREATE TABLE "PORTFOLIO_PROJECTS"(
"UUID" VARCHAR(40) NOT NULL,
"PORTFOLIO_UUID" VARCHAR(40) NOT NULL,
"PROJECT_UUID" VARCHAR(40) NOT NULL,
"CREATED_AT" BIGINT NOT NULL
);
ALTER TABLE "PORTFOLIO_PROJECTS" ADD CONSTRAINT "PK_PORTFOLIO_PROJECTS" PRIMARY KEY("UUID");

CREATE TABLE "PORTFOLIO_REFERENCES"(
"UUID" VARCHAR(40) NOT NULL,
"PORTFOLIO_UUID" VARCHAR(40) NOT NULL,
"REFERENCE_UUID" VARCHAR(40) NOT NULL,
"CREATED_AT" BIGINT NOT NULL
);
ALTER TABLE "PORTFOLIO_REFERENCES" ADD CONSTRAINT "PK_PORTFOLIO_REFERENCES" PRIMARY KEY("UUID");

CREATE TABLE "PORTFOLIOS"(
"UUID" VARCHAR(40) NOT NULL,
"KEE" VARCHAR(400) NOT NULL,
"NAME" VARCHAR(2000) NOT NULL,
"DESCRIPTION" VARCHAR(2000),
"ROOT_UUID" VARCHAR(40) NOT NULL,
"PARENT_UUID" VARCHAR(40),
"PRIVATE" BOOLEAN NOT NULL,
"SELECTION_MODE" VARCHAR(50) NOT NULL,
"SELECTION_EXPRESSION" VARCHAR(50),
"CREATED_AT" BIGINT NOT NULL,
"UPDATED_AT" BIGINT NOT NULL
);
ALTER TABLE "PORTFOLIOS" ADD CONSTRAINT "PK_PORTFOLIOS" PRIMARY KEY("UUID");

CREATE TABLE "PROPERTIES"(
"UUID" VARCHAR(40) NOT NULL,
"PROP_KEY" VARCHAR(512) NOT NULL,
"IS_EMPTY" BOOLEAN NOT NULL,
"TEXT_VALUE" VARCHAR(4000),
"CLOB_VALUE" CLOB,
"CREATED_AT" BIGINT NOT NULL,
"COMPONENT_UUID" VARCHAR(40),
"USER_UUID" VARCHAR(255)
);
ALTER TABLE "PROPERTIES" ADD CONSTRAINT "PK_PROPERTIES" PRIMARY KEY("UUID");
CREATE INDEX "PROPERTIES_KEY" ON "PROPERTIES"("PROP_KEY");

CREATE TABLE "COMPONENTS"(
"UUID" VARCHAR(50) NOT NULL,
"KEE" VARCHAR(400),
"DEPRECATED_KEE" VARCHAR(400),
"NAME" VARCHAR(2000),
"LONG_NAME" VARCHAR(2000),
"DESCRIPTION" VARCHAR(2000),
"ENABLED" BOOLEAN DEFAULT TRUE NOT NULL,
"SCOPE" VARCHAR(3),
"QUALIFIER" VARCHAR(10),
"PRIVATE" BOOLEAN NOT NULL,
"ROOT_UUID" VARCHAR(50) NOT NULL,
"LANGUAGE" VARCHAR(20),
"COPY_COMPONENT_UUID" VARCHAR(50),
"PATH" VARCHAR(2000),
"UUID_PATH" VARCHAR(1500) NOT NULL,
"PROJECT_UUID" VARCHAR(50) NOT NULL,
"MODULE_UUID" VARCHAR(50),
"MODULE_UUID_PATH" VARCHAR(1500),
"MAIN_BRANCH_PROJECT_UUID" VARCHAR(50),
"B_CHANGED" BOOLEAN,
"B_NAME" VARCHAR(500),
"B_LONG_NAME" VARCHAR(500),
"B_DESCRIPTION" VARCHAR(2000),
"B_ENABLED" BOOLEAN,
"B_QUALIFIER" VARCHAR(10),
"B_LANGUAGE" VARCHAR(20),
"B_COPY_COMPONENT_UUID" VARCHAR(50),
"B_PATH" VARCHAR(2000),
"B_UUID_PATH" VARCHAR(1500),
"B_MODULE_UUID" VARCHAR(50),
"B_MODULE_UUID_PATH" VARCHAR(1500),
"CREATED_AT" TIMESTAMP
);
CREATE UNIQUE INDEX "PROJECTS_KEE" ON "COMPONENTS"("KEE");
CREATE INDEX "PROJECTS_MODULE_UUID" ON "COMPONENTS"("MODULE_UUID");
CREATE INDEX "PROJECTS_PROJECT_UUID" ON "COMPONENTS"("PROJECT_UUID");
CREATE INDEX "PROJECTS_QUALIFIER" ON "COMPONENTS"("QUALIFIER");
CREATE INDEX "PROJECTS_ROOT_UUID" ON "COMPONENTS"("ROOT_UUID");
CREATE INDEX "PROJECTS_UUID" ON "COMPONENTS"("UUID");
CREATE INDEX "IDX_MAIN_BRANCH_PRJ_UUID" ON "COMPONENTS"("MAIN_BRANCH_PROJECT_UUID");

+ 27
- 1
server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentUpdater.java View File

@@ -36,6 +36,8 @@ import org.sonar.db.DbSession;
import org.sonar.db.component.BranchDto;
import org.sonar.db.component.BranchType;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.portfolio.PortfolioDto;
import org.sonar.db.portfolio.PortfolioDto.SelectionMode;
import org.sonar.db.project.ProjectDto;
import org.sonar.server.es.ProjectIndexer.Cause;
import org.sonar.server.es.ProjectIndexers;
@@ -90,7 +92,7 @@ public class ComponentUpdater {
* Don't forget to call commitAndIndex(...) when ready to commit.
*/
public ComponentDto createWithoutCommit(DbSession dbSession, NewComponent newComponent,
@Nullable String userUuid, @Nullable String userLogin, Consumer<ComponentDto> componentModifier) {
@Nullable String userUuid, @Nullable String userLogin, Consumer<ComponentDto> componentModifier) {
return createWithoutCommit(dbSession, newComponent, userUuid, userLogin, null, componentModifier);
}

@@ -144,6 +146,12 @@ public class ComponentUpdater {
ProjectDto projectDto = toProjectDto(component, now);
dbClient.projectDao().insert(session, projectDto);
}

if (isRootView(component)) {
PortfolioDto portfolioDto = toPortfolioDto(component, now);
dbClient.portfolioDao().insert(session, portfolioDto);
}

return component;
}

@@ -159,11 +167,29 @@ public class ComponentUpdater {
.setCreatedAt(now);
}

private static PortfolioDto toPortfolioDto(ComponentDto component, long now) {
return new PortfolioDto()
.setUuid(component.uuid())
.setRootUuid(component.projectUuid())
.setKey(component.getKey())
.setName(component.name())
.setPrivate(component.isPrivate())
.setDescription(component.description())
.setSelectionMode(SelectionMode.NONE.name())
.setUpdatedAt(now)
.setCreatedAt(now);
}

private static boolean isRootProject(ComponentDto componentDto) {
return Scopes.PROJECT.equals(componentDto.scope())
&& MAIN_BRANCH_QUALIFIERS.contains(componentDto.qualifier());
}

private static boolean isRootView(ComponentDto componentDto) {
return Scopes.PROJECT.equals(componentDto.scope())
&& Qualifiers.VIEW.contains(componentDto.qualifier());
}

private void createMainBranch(DbSession session, String componentUuid, @Nullable String mainBranch) {
BranchDto branch = new BranchDto()
.setBranchType(BranchType.BRANCH)

Write Preview
Loading…
Cancel
Save