From 07054006cb435f38b30336133cd8e37d9767619e Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Thu, 6 Dec 2018 08:13:23 -0600 Subject: [PATCH] SONAR-11464 Fix migration of branches and add IT --- .../component/ComponentUuidFactory.java | 107 +------------- .../component/ComponentUuidFactoryImpl.java | 45 ++++++ .../ComponentUuidFactoryWithMigration.java | 133 ++++++++++++++++++ .../step/BuildComponentTreeStep.java | 11 +- ...omponentUuidFactoryWithMigrationTest.java} | 54 ++++++- .../step/BuildComponentTreeStepTest.java | 32 +++-- .../db/component/ComponentKeyUpdaterDao.java | 8 +- .../server/component/ComponentService.java | 4 +- .../server/component/ComponentUpdater.java | 4 +- .../sonar/core/component/ComponentKeys.java | 70 ++++----- .../core/component/ComponentKeysTest.java | 36 ++--- .../scanner/scan/ProjectReactorValidator.java | 4 +- 12 files changed, 318 insertions(+), 190 deletions(-) create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryWithMigration.java rename server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/{ComponentUuidFactoryTest.java => ComponentUuidFactoryWithMigrationTest.java} (64%) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactory.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactory.java index f42749ae0c9..76545a33367 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactory.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactory.java @@ -19,112 +19,9 @@ */ package org.sonar.ce.task.projectanalysis.component; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; -import org.apache.commons.lang.StringUtils; -import org.sonar.api.resources.Qualifiers; -import org.sonar.api.resources.Scopes; -import org.sonar.core.component.ComponentKeys; -import org.sonar.core.util.Uuids; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentWithModuleUuidDto; -import org.sonar.db.component.KeyWithUuidDto; - -public class ComponentUuidFactory { - private final Map uuidsByKey = new HashMap<>(); - - public ComponentUuidFactory(DbClient dbClient, DbSession dbSession, String rootKey, Map reportModulesPath) { - Map modulePathsByUuid; - - if (reportModulesPath.isEmpty()) { - noMigration(dbClient, dbSession, rootKey); - } else { - modulePathsByUuid = loadModulePathsByUuid(dbClient, dbSession, rootKey, reportModulesPath); - - if (modulePathsByUuid.isEmpty()) { - noMigration(dbClient, dbSession, rootKey); - } else { - doMigration(dbClient, dbSession, rootKey, modulePathsByUuid); - } - } - } - - private void noMigration(DbClient dbClient, DbSession dbSession, String rootKey) { - List keys = dbClient.componentDao().selectUuidsByKeyFromProjectKey(dbSession, rootKey); - keys.forEach(dto -> uuidsByKey.put(dto.key(), dto.uuid())); - } - - private void doMigration(DbClient dbClient, DbSession dbSession, String rootKey, Map modulePathsByUuid) { - List dtos = loadComponentsWithModuleUuid(dbClient, dbSession, rootKey); - for (ComponentWithModuleUuidDto dto : dtos) { - if ("/".equals(dto.path())) { - // skip root folders - continue; - } - - if (Scopes.PROJECT.equals(dto.scope())) { - String modulePathFromRootProject = modulePathsByUuid.get(dto.uuid()); - if (modulePathFromRootProject != null || StringUtils.isEmpty(dto.moduleUuid())) { - // means that it's a root or a module with a valid path (to avoid overwriting key of root) - uuidsByKey.put(ComponentKeys.createEffectiveKey(rootKey, modulePathFromRootProject), dto.uuid()); - } - } else { - String modulePathFromRootProject = modulePathsByUuid.get(dto.moduleUuid()); - String componentPath = createComponentPath(dto, modulePathFromRootProject); - uuidsByKey.put(ComponentKeys.createEffectiveKey(rootKey, componentPath), dto.uuid()); - } - } - } - - @CheckForNull - private static String createComponentPath(ComponentWithModuleUuidDto dto, @Nullable String modulePathFromRootProject) { - if (StringUtils.isEmpty(modulePathFromRootProject)) { - return dto.path(); - } - - if (StringUtils.isEmpty(dto.path())) { - // will be the case for modules - return modulePathFromRootProject; - } - - return modulePathFromRootProject + "/" + dto.path(); - } - - private static List loadComponentsWithModuleUuid(DbClient dbClient, DbSession dbSession, String rootKey) { - return dbClient.componentDao().selectComponentsWithModuleUuidFromProjectKey(dbSession, rootKey); - } - - private static Map loadModulePathsByUuid(DbClient dbClient, DbSession dbSession, String rootKey, Map pathByModuleKey) { - List moduleDtos = dbClient.componentDao() - .selectModulesFromProjectKey(dbSession, rootKey, false).stream() - .filter(c -> Qualifiers.MODULE.equals(c.qualifier())) - .collect(Collectors.toList()); - - if (moduleDtos.isEmpty()) { - return Collections.emptyMap(); - } - - Map modulePathByUuid = new HashMap<>(); - for (ComponentDto dto : moduleDtos) { - String relativePath = pathByModuleKey.get(dto.getKey()); - if (relativePath != null) { - modulePathByUuid.put(dto.uuid(), relativePath); - } - } - return modulePathByUuid; - } - +public interface ComponentUuidFactory { /** * Get UUID from database if it exists, otherwise generate a new one. */ - public String getOrCreateForKey(String key) { - return uuidsByKey.computeIfAbsent(key, k -> Uuids.create()); - } + String getOrCreateForKey(String key); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryImpl.java new file mode 100644 index 00000000000..f6780d8f3c6 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryImpl.java @@ -0,0 +1,45 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.sonar.core.util.Uuids; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.KeyWithUuidDto; + +public class ComponentUuidFactoryImpl implements ComponentUuidFactory { + private final Map uuidsByKey = new HashMap<>(); + + public ComponentUuidFactoryImpl(DbClient dbClient, DbSession dbSession, String rootKey) { + List keys = dbClient.componentDao().selectUuidsByKeyFromProjectKey(dbSession, rootKey); + keys.forEach(dto -> uuidsByKey.put(dto.key(), dto.uuid())); + } + + /** + * Get UUID from database if it exists, otherwise generate a new one. + */ + @Override + public String getOrCreateForKey(String key) { + return uuidsByKey.computeIfAbsent(key, k -> Uuids.create()); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryWithMigration.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryWithMigration.java new file mode 100644 index 00000000000..732f278d20a --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryWithMigration.java @@ -0,0 +1,133 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.component; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import org.apache.commons.lang.StringUtils; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.core.component.ComponentKeys; +import org.sonar.core.util.Uuids; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentWithModuleUuidDto; +import org.sonar.db.component.KeyWithUuidDto; + +public class ComponentUuidFactoryWithMigration implements ComponentUuidFactory { + private final Map uuidsByKey = new HashMap<>(); + + public ComponentUuidFactoryWithMigration(DbClient dbClient, DbSession dbSession, String rootKey, Function pathToKey, Map reportModulesPath) { + Map modulePathsByUuid; + + if (reportModulesPath.isEmpty()) { + noMigration(dbClient, dbSession, rootKey); + } else { + modulePathsByUuid = loadModulePathsByUuid(dbClient, dbSession, rootKey, reportModulesPath); + + if (modulePathsByUuid.isEmpty()) { + noMigration(dbClient, dbSession, rootKey); + } else { + doMigration(dbClient, dbSession, rootKey, pathToKey, modulePathsByUuid); + } + } + } + + private void noMigration(DbClient dbClient, DbSession dbSession, String rootKey) { + List keys = dbClient.componentDao().selectUuidsByKeyFromProjectKey(dbSession, rootKey); + keys.forEach(dto -> uuidsByKey.put(dto.key(), dto.uuid())); + } + + private void doMigration(DbClient dbClient, DbSession dbSession, String rootKey, Function pathToKey, Map modulePathsByUuid) { + List dtos = loadComponentsWithModuleUuid(dbClient, dbSession, rootKey); + for (ComponentWithModuleUuidDto dto : dtos) { + if ("/".equals(dto.path())) { + // skip root folders + continue; + } + + if (Scopes.PROJECT.equals(dto.scope())) { + String modulePathFromRootProject = modulePathsByUuid.get(dto.uuid()); + if (modulePathFromRootProject != null || StringUtils.isEmpty(dto.moduleUuid())) { + // means that it's a root or a module with a valid path (to avoid overwriting key of root) + pathToKey.apply(modulePathFromRootProject); + uuidsByKey.put(pathToKey.apply(modulePathFromRootProject), dto.uuid()); + } + } else { + String modulePathFromRootProject = modulePathsByUuid.get(dto.moduleUuid()); + String componentPath = createComponentPath(dto, modulePathFromRootProject); + uuidsByKey.put(pathToKey.apply(componentPath), dto.uuid()); + } + } + } + + @CheckForNull + private static String createComponentPath(ComponentWithModuleUuidDto dto, @Nullable String modulePathFromRootProject) { + if (StringUtils.isEmpty(modulePathFromRootProject)) { + return dto.path(); + } + + if (StringUtils.isEmpty(dto.path())) { + // will be the case for modules + return modulePathFromRootProject; + } + + return modulePathFromRootProject + "/" + dto.path(); + } + + private static List loadComponentsWithModuleUuid(DbClient dbClient, DbSession dbSession, String rootKey) { + return dbClient.componentDao().selectComponentsWithModuleUuidFromProjectKey(dbSession, rootKey); + } + + private static Map loadModulePathsByUuid(DbClient dbClient, DbSession dbSession, String rootKey, Map pathByModuleKey) { + List moduleDtos = dbClient.componentDao() + .selectModulesFromProjectKey(dbSession, rootKey, false).stream() + .filter(c -> Qualifiers.MODULE.equals(c.qualifier())) + .collect(Collectors.toList()); + + if (moduleDtos.isEmpty()) { + return Collections.emptyMap(); + } + + Map modulePathByUuid = new HashMap<>(); + for (ComponentDto dto : moduleDtos) { + String relativePath = pathByModuleKey.get(dto.getKey()); + if (relativePath != null) { + modulePathByUuid.put(dto.uuid(), relativePath); + } + } + return modulePathByUuid; + } + + /** + * Get UUID from database if it exists, otherwise generate a new one. + */ + @Override + public String getOrCreateForKey(String key) { + return uuidsByKey.computeIfAbsent(key, k -> Uuids.create()); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java index 9bd4a18fac5..b7f343c477f 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStep.java @@ -19,6 +19,7 @@ */ package org.sonar.ce.task.projectanalysis.step; +import java.util.function.Function; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.ce.task.projectanalysis.analysis.Analysis; @@ -28,7 +29,7 @@ import org.sonar.ce.task.projectanalysis.batch.BatchReportReader; import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.ComponentKeyGenerator; import org.sonar.ce.task.projectanalysis.component.ComponentTreeBuilder; -import org.sonar.ce.task.projectanalysis.component.ComponentUuidFactory; +import org.sonar.ce.task.projectanalysis.component.ComponentUuidFactoryWithMigration; import org.sonar.ce.task.projectanalysis.component.DefaultBranchImpl; import org.sonar.ce.task.projectanalysis.component.MutableTreeRootHolder; import org.sonar.ce.task.projectanalysis.component.ReportModulesPath; @@ -77,15 +78,15 @@ public class BuildComponentTreeStep implements ComputationStep { // root key of branch, not necessarily of project String rootKey = keyGenerator.generateKey(reportProject, null); - + Function pathToKey = path -> keyGenerator.generateKey(reportProject, path); // loads the UUIDs from database. If they don't exist, then generate new ones - ComponentUuidFactory componentUuidFactory = new ComponentUuidFactory(dbClient, dbSession, rootKey, reportModulesPath.get()); + ComponentUuidFactoryWithMigration componentUuidFactoryWithMigration = new ComponentUuidFactoryWithMigration(dbClient, dbSession, rootKey, pathToKey, reportModulesPath.get()); - String rootUuid = componentUuidFactory.getOrCreateForKey(rootKey); + String rootUuid = componentUuidFactoryWithMigration.getOrCreateForKey(rootKey); SnapshotDto baseAnalysis = loadBaseAnalysis(dbSession, rootUuid); ComponentTreeBuilder builder = new ComponentTreeBuilder(keyGenerator, publicKeyGenerator, - componentUuidFactory::getOrCreateForKey, + componentUuidFactoryWithMigration::getOrCreateForKey, reportReader::readComponent, analysisMetadataHolder.getProject(), analysisMetadataHolder.getBranch(), diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryWithMigrationTest.java similarity index 64% rename from server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryTest.java rename to server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryWithMigrationTest.java index 96c54c65be8..05b6dd760a2 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/ComponentUuidFactoryWithMigrationTest.java @@ -22,6 +22,7 @@ package org.sonar.ce.task.projectanalysis.component; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.System2; @@ -31,17 +32,20 @@ import org.sonar.db.component.ComponentTesting; import static org.assertj.core.api.Assertions.assertThat; -public class ComponentUuidFactoryTest { +public class ComponentUuidFactoryWithMigrationTest { @Rule public DbTester db = DbTester.create(System2.INSTANCE); + private Function pathToKey = path -> path != null ? "project:" + path : "project"; @Test public void load_uuids_from_existing_components_in_db() { ComponentDto project = db.components().insertPrivateProject(); ComponentDto module = db.components().insertComponent(ComponentTesting.newModuleDto(project)); Map reportModulesPath = Collections.singletonMap(module.getKey(), "module1_path"); - ComponentUuidFactory underTest = new ComponentUuidFactory(db.getDbClient(), db.getSession(), project.getDbKey(), reportModulesPath); + pathToKey = path -> path != null ? project.getDbKey() + ":" + path : project.getDbKey(); + + ComponentUuidFactoryWithMigration underTest = new ComponentUuidFactoryWithMigration(db.getDbClient(), db.getSession(), project.getDbKey(), pathToKey, reportModulesPath); assertThat(underTest.getOrCreateForKey(project.getDbKey())).isEqualTo(project.uuid()); assertThat(underTest.getOrCreateForKey(module.getDbKey())).isNotIn(project.uuid(), module.uuid()); @@ -65,7 +69,7 @@ public class ComponentUuidFactoryTest { Map modulesRelativePaths = new HashMap<>(); modulesRelativePaths.put("project:module1", "module1_path"); modulesRelativePaths.put("project:module1:module2", "module1_path/module2_path"); - ComponentUuidFactory underTest = new ComponentUuidFactory(db.getDbClient(), db.getSession(), project.getDbKey(), modulesRelativePaths); + ComponentUuidFactoryWithMigration underTest = new ComponentUuidFactoryWithMigration(db.getDbClient(), db.getSession(), project.getDbKey(), pathToKey, modulesRelativePaths); // migrated files assertThat(underTest.getOrCreateForKey("project:file1_path")).isEqualTo(file1.uuid()); @@ -81,6 +85,42 @@ public class ComponentUuidFactoryTest { assertThat(underTest.getOrCreateForKey(file2.getDbKey())).isNotIn(project.uuid(), module1.uuid(), module2.uuid(), file1.uuid(), file2.uuid()); } + @Test + public void migrate_branch_with_modules() { + pathToKey = path -> path != null ? "project:" + path + ":BRANCH:branch1" : "project:BRANCH:branch1"; + ComponentDto project = db.components().insertPrivateProject(dto -> dto.setDbKey("project:BRANCH:branch1")); + ComponentDto module1 = db.components().insertComponent(ComponentTesting.newModuleDto(project) + .setDbKey("project:module1:BRANCH:branch1")); + ComponentDto module2 = db.components().insertComponent(ComponentTesting.newModuleDto(module1) + .setDbKey("project:module1:module2:BRANCH:branch1")); + ComponentDto file1 = db.components().insertComponent(ComponentTesting.newFileDto(project) + .setDbKey("project:file1:BRANCH:branch1") + .setPath("file1_path")); + ComponentDto file2 = db.components().insertComponent(ComponentTesting.newFileDto(module2) + .setDbKey("project:module1:module2:file2:BRANCH:branch1") + .setPath("file2_path")); + + assertThat(file2.moduleUuidPath()).isEqualTo("." + project.uuid() + "." + module1.uuid() + "." + module2.uuid() + "."); + Map modulesRelativePaths = new HashMap<>(); + modulesRelativePaths.put("project:module1", "module1_path"); + modulesRelativePaths.put("project:module1:module2", "module1_path/module2_path"); + ComponentUuidFactoryWithMigration underTest = new ComponentUuidFactoryWithMigration(db.getDbClient(), db.getSession(), project.getDbKey(), pathToKey, modulesRelativePaths); + + // migrated files + assertThat(underTest.getOrCreateForKey("project:file1_path:BRANCH:branch1")).isEqualTo(file1.uuid()); + assertThat(underTest.getOrCreateForKey("project:module1_path/module2_path/file2_path:BRANCH:branch1")).isEqualTo(file2.uuid()); + + // project remains the same + assertThat(underTest.getOrCreateForKey(project.getDbKey())).isEqualTo(project.uuid()); + + // old keys with modules don't exist + assertThat(underTest.getOrCreateForKey(module1.getDbKey())).isNotIn(project.uuid(), module1.uuid(), module2.uuid(), file1.uuid(), file2.uuid()); + assertThat(underTest.getOrCreateForKey(module2.getDbKey())).isNotIn(project.uuid(), module1.uuid(), module2.uuid(), file1.uuid(), file2.uuid()); + assertThat(underTest.getOrCreateForKey(file1.getDbKey())).isNotIn(project.uuid(), module1.uuid(), module2.uuid(), file1.uuid(), file2.uuid()); + assertThat(underTest.getOrCreateForKey(file2.getDbKey())).isNotIn(project.uuid(), module1.uuid(), module2.uuid(), file1.uuid(), file2.uuid()); + + } + @Test public void migrate_project_with_root_folders() { ComponentDto project = db.components().insertPrivateProject(dto -> dto.setDbKey("project")); @@ -90,7 +130,7 @@ public class ComponentUuidFactoryTest { .setDbKey("project:module1:/")); Map modulesRelativePaths = Collections.singletonMap("project:module1", "module1_path"); - ComponentUuidFactory underTest = new ComponentUuidFactory(db.getDbClient(), db.getSession(), project.getDbKey(), modulesRelativePaths); + ComponentUuidFactoryWithMigration underTest = new ComponentUuidFactoryWithMigration(db.getDbClient(), db.getSession(), project.getDbKey(), pathToKey, modulesRelativePaths); // project remains the same assertThat(underTest.getOrCreateForKey(project.getDbKey())).isEqualTo(project.uuid()); @@ -115,7 +155,7 @@ public class ComponentUuidFactoryTest { .setPath("file1_path")); Map modulesRelativePaths = Collections.singletonMap("project:module1", "module1_path"); - ComponentUuidFactory underTest = new ComponentUuidFactory(db.getDbClient(), db.getSession(), project.getDbKey(), modulesRelativePaths); + ComponentUuidFactoryWithMigration underTest = new ComponentUuidFactoryWithMigration(db.getDbClient(), db.getSession(), project.getDbKey(), pathToKey, modulesRelativePaths); // migrated file assertThat(underTest.getOrCreateForKey("project:module1_path/file1_path")).isEqualTo(file1.uuid()); @@ -133,7 +173,7 @@ public class ComponentUuidFactoryTest { Map modulesRelativePaths = new HashMap<>(); modulesRelativePaths.put("project", ""); modulesRelativePaths.put("project:module2", "module2"); - ComponentUuidFactory underTest = new ComponentUuidFactory(db.getDbClient(), db.getSession(), project.getDbKey(), modulesRelativePaths); + ComponentUuidFactoryWithMigration underTest = new ComponentUuidFactoryWithMigration(db.getDbClient(), db.getSession(), project.getDbKey(), pathToKey, modulesRelativePaths); // check root project. assertThat(underTest.getOrCreateForKey("project")).isEqualTo(project.uuid()); @@ -141,7 +181,7 @@ public class ComponentUuidFactoryTest { @Test public void generate_uuid_if_it_does_not_exist_in_db() { - ComponentUuidFactory underTest = new ComponentUuidFactory(db.getDbClient(), db.getSession(), "theProjectKey", Collections.emptyMap()); + ComponentUuidFactoryWithMigration underTest = new ComponentUuidFactoryWithMigration(db.getDbClient(), db.getSession(), "theProjectKey", pathToKey, Collections.emptyMap()); String generatedKey = underTest.getOrCreateForKey("foo"); assertThat(generatedKey).isNotEmpty(); diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java index 5d7b05fc7c5..56d4d24a09b 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/BuildComponentTreeStepTest.java @@ -437,15 +437,22 @@ public class BuildComponentTreeStepTest { public void return_existing_uuids_when_components_were_removed() { setAnalysisMetadataHolder(); OrganizationDto organizationDto = dbTester.organizations().insert(); - ComponentDto project = insertComponent(newPrivateProjectDto(organizationDto, "ABCD").setDbKey(REPORT_PROJECT_KEY)); - ComponentDto removedModule = insertComponent(newModuleDto("BCDE", project).setDbKey(REPORT_MODULE_KEY).setEnabled(false)); - ComponentDto removedDirectory = insertComponent(newDirectory(removedModule, "CDEF", REPORT_DIR_PATH_1).setDbKey(REPORT_MODULE_KEY + ":" + REPORT_DIR_PATH_1).setEnabled(false)); - insertComponent(newFileDto(removedModule, removedDirectory, "DEFG").setDbKey(REPORT_MODULE_KEY + ":" + REPORT_FILE_PATH_1).setEnabled(false)); + ComponentDto project = insertComponent(newPrivateProjectDto(organizationDto, "ABCD") + .setDbKey(REPORT_PROJECT_KEY)); + ComponentDto removedModule = insertComponent(newModuleDto("BCDE", project) + .setDbKey(REPORT_MODULE_KEY).setEnabled(false)); + ComponentDto removedDirectory = insertComponent(newDirectory(removedModule, "CDEF", REPORT_DIR_PATH_1) + .setDbKey(REPORT_MODULE_KEY + ":" + REPORT_DIR_PATH_1).setEnabled(false)); + insertComponent(newFileDto(removedModule, removedDirectory, "DEFG") + .setDbKey(REPORT_MODULE_KEY + ":" + REPORT_FILE_PATH_1).setPath(REPORT_FILE_PATH_1).setEnabled(false)); + + reportReader.putComponent(component(ROOT_REF, PROJECT, REPORT_PROJECT_KEY, FILE_1_REF)); + reportReader.putComponent(componentWithPath(FILE_1_REF, FILE, "module/" + REPORT_FILE_PATH_1)); - reportReader.putComponent(component(ROOT_REF, PROJECT, REPORT_PROJECT_KEY, MODULE_REF)); - reportReader.putComponent(component(MODULE_REF, MODULE, REPORT_MODULE_KEY, DIR_REF_1)); - reportReader.putComponent(componentWithPath(DIR_REF_1, DIRECTORY, REPORT_DIR_PATH_1, FILE_1_REF)); - reportReader.putComponent(componentWithPath(FILE_1_REF, FILE, REPORT_FILE_PATH_1)); + reportReader.setMetadata(ScannerReport.Metadata.newBuilder() + .putModulesProjectRelativePathByKey(REPORT_PROJECT_KEY, "") + .putModulesProjectRelativePathByKey(REPORT_MODULE_KEY, "module") + .build()); underTest.execute(new TestComputationStepContext()); @@ -453,9 +460,8 @@ public class BuildComponentTreeStepTest { // No new UUID is generated on removed components verifyComponentMissingByRef(MODULE_REF); - // TODO migrate modules - //verifyComponentByRef(DIR_REF_1, REPORT_PROJECT_KEY + ":" + REPORT_DIR_PATH_1, REPORT_PROJECT_KEY + ":" + REPORT_DIR_PATH_1, "CDEF"); - //verifyComponentByRef(FILE_1_REF, REPORT_PROJECT_KEY + ":" + REPORT_FILE_PATH_1, "DEFG"); + verifyComponentByKey(REPORT_PROJECT_KEY + ":module/" + REPORT_DIR_PATH_1, REPORT_PROJECT_KEY + ":module/" + REPORT_DIR_PATH_1, "CDEF"); + verifyComponentByRef(FILE_1_REF, REPORT_PROJECT_KEY + ":module/" + REPORT_FILE_PATH_1, "DEFG"); } @Test @@ -594,7 +600,9 @@ public class BuildComponentTreeStepTest { } private static void feedComponentByRef(Component component, Map map) { - map.put(component.getReportAttributes().getRef(), component); + if (component.getReportAttributes().getRef() != null) { + map.put(component.getReportAttributes().getRef(), component); + } for (Component child : component.getChildren()) { feedComponentByRef(child, map); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentKeyUpdaterDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentKeyUpdaterDao.java index dc0c97814a7..84d686173c9 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentKeyUpdaterDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentKeyUpdaterDao.java @@ -41,8 +41,8 @@ import org.sonar.db.Dao; import org.sonar.db.DbSession; import static com.google.common.base.Preconditions.checkArgument; -import static org.sonar.core.component.ComponentKeys.checkModuleKey; -import static org.sonar.core.component.ComponentKeys.isValidModuleKey; +import static org.sonar.core.component.ComponentKeys.checkProjectKey; +import static org.sonar.core.component.ComponentKeys.isValidProjectKey; /** * Class used to rename the key of a project and its resources. @@ -94,7 +94,7 @@ public class ComponentKeyUpdaterDao implements Dao { ResourceDto::getKey, component -> { String newKey = computeNewKey(component.getKey(), stringToReplace, replacementString); - checkModuleKey(newKey); + checkProjectKey(newKey); return newKey; })); } @@ -233,7 +233,7 @@ public class ComponentKeyUpdaterDao implements Dao { private static void checkNewNameOfAllModules(Set modules, String stringToReplace, String replacementString, ComponentKeyUpdaterMapper mapper) { for (ResourceDto module : modules) { String newKey = computeNewKey(module.getKey(), stringToReplace, replacementString); - checkArgument(isValidModuleKey(newKey), "Malformed key for '%s'. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", newKey); + checkArgument(isValidProjectKey(newKey), "Malformed key for '%s'. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", newKey); if (mapper.countResourceByKey(newKey) > 0) { throw new IllegalArgumentException("Impossible to update key: a component with key \"" + newKey + "\" already exists."); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentService.java b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentService.java index 7d60d1b288b..f72e0c6803c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentService.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentService.java @@ -40,7 +40,7 @@ import org.sonar.server.user.UserSession; import static java.util.Collections.emptyList; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -import static org.sonar.core.component.ComponentKeys.isValidModuleKey; +import static org.sonar.core.component.ComponentKeys.isValidProjectKey; import static org.sonar.db.component.ComponentKeyUpdaterDao.checkIsProjectOrModule; import static org.sonar.server.ws.WsUtils.checkRequest; @@ -102,7 +102,7 @@ public class ComponentService { } private static void checkProjectOrModuleKeyFormat(String key) { - checkRequest(isValidModuleKey(key), "Malformed key for '%s'. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", key); + checkRequest(isValidProjectKey(key), "Malformed key for '%s'. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", key); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentUpdater.java b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentUpdater.java index 5ec5c494bdd..8c784b3ad2b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentUpdater.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentUpdater.java @@ -43,7 +43,7 @@ import org.sonar.server.permission.PermissionTemplateService; import static java.util.Collections.singletonList; import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.core.component.ComponentKeys.isValidModuleKey; +import static org.sonar.core.component.ComponentKeys.isValidProjectKey; import static org.sonar.server.ws.WsUtils.checkRequest; public class ComponentUpdater { @@ -165,7 +165,7 @@ public class ComponentUpdater { } private void checkKeyFormat(String qualifier, String key) { - checkRequest(isValidModuleKey(key), + checkRequest(isValidProjectKey(key), "Malformed key for %s: %s. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", getQualifierToDisplay(qualifier), key); } diff --git a/sonar-core/src/main/java/org/sonar/core/component/ComponentKeys.java b/sonar-core/src/main/java/org/sonar/core/component/ComponentKeys.java index 440f7339f1f..79534000e30 100644 --- a/sonar-core/src/main/java/org/sonar/core/component/ComponentKeys.java +++ b/sonar-core/src/main/java/org/sonar/core/component/ComponentKeys.java @@ -32,9 +32,9 @@ public final class ComponentKeys { /* * Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit */ - private static final String VALID_MODULE_KEY_REGEXP = "[\\p{Alnum}\\-_.:]*[\\p{Alpha}\\-_.:]+[\\p{Alnum}\\-_.:]*"; + private static final String VALID_PROJECT_KEY_REGEXP = "[\\p{Alnum}\\-_.:]*[\\p{Alpha}\\-_.:]+[\\p{Alnum}\\-_.:]*"; - private static final String VALID_MODULE_KEY_ISSUES_MODE_REGEXP = "[\\p{Alnum}\\-_.:/]*[\\p{Alpha}\\-_.:/]+[\\p{Alnum}\\-_.:/]*"; + private static final String VALID_PROJECT_KEY_ISSUES_MODE_REGEXP = "[\\p{Alnum}\\-_.:/]*[\\p{Alpha}\\-_.:/]+[\\p{Alnum}\\-_.:/]*"; /* * Allowed characters are alphanumeric, '-', '_', '.' and '/' */ @@ -60,62 +60,66 @@ public final class ComponentKeys { } /** - *

Test if given parameter is valid for a project/module. Valid format is:

+ *

Test if given parameter is valid for a project. Valid format is:

*
    - *
  • Allowed characters: - *
      - *
    • Uppercase ASCII letters A-Z
      • - *
    • Lowercase ASCII letters a-z
      • - *
    • ASCII digits 0-9
      • - *
    • Punctuation signs dash '-', underscore '_', period '.' and colon ':'
      • - *
    - *
    • - *
  • At least one non-digit
    • + *
  • Allowed characters: + *
      + *
    • Uppercase ASCII letters A-Z
      • + *
    • Lowercase ASCII letters a-z
      • + *
    • ASCII digits 0-9
      • + *
    • Punctuation signs dash '-', underscore '_', period '.' and colon ':'
      • + *
    + *
    • + *
  • At least one non-digit
    • *
+ * * @param keyCandidate - * @return true if keyCandidate can be used for a project/module + * @return true if keyCandidate can be used for a project */ - public static boolean isValidModuleKey(String keyCandidate) { - return keyCandidate.matches(VALID_MODULE_KEY_REGEXP); + public static boolean isValidProjectKey(String keyCandidate) { + return keyCandidate.matches(VALID_PROJECT_KEY_REGEXP); } /** - * Checks if given parameter is valid for a project/module following {@link #isValidModuleKey(String)} contract. + * Checks if given parameter is valid for a project following {@link #isValidProjectKey(String)} contract. * * @throws IllegalArgumentException if the format is incorrect */ - public static void checkModuleKey(String keyCandidate) { - checkArgument(isValidModuleKey(keyCandidate), "Malformed key for '%s'. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", keyCandidate); + public static void checkProjectKey(String keyCandidate) { + checkArgument(isValidProjectKey(keyCandidate), "Malformed key for '%s'. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", + keyCandidate); } /** - * Same as {@link #isValidModuleKey(String)}, but allows additionally '/'. + * Same as {@link #isValidProjectKey(String)}, but allows additionally '/'. */ - public static boolean isValidModuleKeyIssuesMode(String keyCandidate) { - return keyCandidate.matches(VALID_MODULE_KEY_ISSUES_MODE_REGEXP); + public static boolean isValidProjectKeyIssuesMode(String keyCandidate) { + return keyCandidate.matches(VALID_PROJECT_KEY_ISSUES_MODE_REGEXP); } /** *

Test if given parameter is valid for a branch. Valid format is:

*
    - *
  • Allowed characters: - *
      - *
    • Uppercase ASCII letters A-Z
      • - *
    • Lowercase ASCII letters a-z
      • - *
    • ASCII digits 0-9
      • - *
    • Punctuation signs dash '-', underscore '_', period '.', and '/'
      • - *
    - *
    • + *
  • Allowed characters: + *
      + *
    • Uppercase ASCII letters A-Z
      • + *
    • Lowercase ASCII letters a-z
      • + *
    • ASCII digits 0-9
      • + *
    • Punctuation signs dash '-', underscore '_', period '.', and '/'
      • *
    + *
    • + *
+ * * @param branchCandidate - * @return true if branchCandidate can be used for a project/module + * @return true if branchCandidate can be used for a project */ public static boolean isValidBranch(String branchCandidate) { return branchCandidate.matches(VALID_BRANCH_REGEXP); } /** - * Return the project/module key with potential branch + * Return the project key with potential branch + * * @param keyWithoutBranch * @param branch * @return @@ -128,8 +132,8 @@ public final class ComponentKeys { } } - public static String createKey(String moduleKey, @Nullable String path, @Nullable String branch) { - String key = createKey(moduleKey, branch); + public static String createKey(String projectKey, @Nullable String path, @Nullable String branch) { + String key = createKey(projectKey, branch); return createEffectiveKey(key, path); } } diff --git a/sonar-core/src/test/java/org/sonar/core/component/ComponentKeysTest.java b/sonar-core/src/test/java/org/sonar/core/component/ComponentKeysTest.java index f2104fbcab4..d4f45a69769 100644 --- a/sonar-core/src/test/java/org/sonar/core/component/ComponentKeysTest.java +++ b/sonar-core/src/test/java/org/sonar/core/component/ComponentKeysTest.java @@ -48,22 +48,22 @@ public class ComponentKeysTest { @Test public void isValidModuleKey() { - assertThat(ComponentKeys.isValidModuleKey("")).isFalse(); - assertThat(ComponentKeys.isValidModuleKey("abc")).isTrue(); - assertThat(ComponentKeys.isValidModuleKey("0123")).isFalse(); - assertThat(ComponentKeys.isValidModuleKey("ab 12")).isFalse(); - assertThat(ComponentKeys.isValidModuleKey("ab_12")).isTrue(); - assertThat(ComponentKeys.isValidModuleKey("ab/12")).isFalse(); + assertThat(ComponentKeys.isValidProjectKey("")).isFalse(); + assertThat(ComponentKeys.isValidProjectKey("abc")).isTrue(); + assertThat(ComponentKeys.isValidProjectKey("0123")).isFalse(); + assertThat(ComponentKeys.isValidProjectKey("ab 12")).isFalse(); + assertThat(ComponentKeys.isValidProjectKey("ab_12")).isTrue(); + assertThat(ComponentKeys.isValidProjectKey("ab/12")).isFalse(); } @Test public void isValidModuleKeyIssuesMode() { - assertThat(ComponentKeys.isValidModuleKeyIssuesMode("")).isFalse(); - assertThat(ComponentKeys.isValidModuleKeyIssuesMode("abc")).isTrue(); - assertThat(ComponentKeys.isValidModuleKeyIssuesMode("0123")).isFalse(); - assertThat(ComponentKeys.isValidModuleKeyIssuesMode("ab 12")).isFalse(); - assertThat(ComponentKeys.isValidModuleKeyIssuesMode("ab_12")).isTrue(); - assertThat(ComponentKeys.isValidModuleKeyIssuesMode("ab/12")).isTrue(); + assertThat(ComponentKeys.isValidProjectKeyIssuesMode("")).isFalse(); + assertThat(ComponentKeys.isValidProjectKeyIssuesMode("abc")).isTrue(); + assertThat(ComponentKeys.isValidProjectKeyIssuesMode("0123")).isFalse(); + assertThat(ComponentKeys.isValidProjectKeyIssuesMode("ab 12")).isFalse(); + assertThat(ComponentKeys.isValidProjectKeyIssuesMode("ab_12")).isTrue(); + assertThat(ComponentKeys.isValidProjectKeyIssuesMode("ab/12")).isTrue(); } @Test @@ -80,8 +80,8 @@ public class ComponentKeysTest { @Test public void checkModuleKey_with_correct_keys() { - ComponentKeys.checkModuleKey("abc"); - ComponentKeys.checkModuleKey("a-b_1.:2"); + ComponentKeys.checkProjectKey("abc"); + ComponentKeys.checkProjectKey("a-b_1.:2"); } @Test @@ -89,27 +89,27 @@ public class ComponentKeysTest { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Malformed key for '0123'. Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit."); - ComponentKeys.checkModuleKey("0123"); + ComponentKeys.checkProjectKey("0123"); } @Test public void checkModuleKey_fail_if_key_is_empty() { expectedException.expect(IllegalArgumentException.class); - ComponentKeys.checkModuleKey(""); + ComponentKeys.checkProjectKey(""); } @Test public void checkModuleKey_fail_if_space() { expectedException.expect(IllegalArgumentException.class); - ComponentKeys.checkModuleKey("ab 12"); + ComponentKeys.checkProjectKey("ab 12"); } @Test public void checkModuleKey_fail_if_special_characters_not_allowed() { expectedException.expect(IllegalArgumentException.class); - ComponentKeys.checkModuleKey("ab/12"); + ComponentKeys.checkProjectKey("ab/12"); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorValidator.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorValidator.java index 6c719052863..5eba211bbca 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorValidator.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorValidator.java @@ -109,14 +109,14 @@ public class ProjectReactorValidator { } private static void validateModuleIssuesMode(ProjectDefinition moduleDef, List validationMessages) { - if (!ComponentKeys.isValidModuleKeyIssuesMode(moduleDef.getKey())) { + if (!ComponentKeys.isValidProjectKeyIssuesMode(moduleDef.getKey())) { validationMessages.add(format("\"%s\" is not a valid project or module key. " + "Allowed characters in issues mode are alphanumeric, '-', '_', '.', '/' and ':', with at least one non-digit.", moduleDef.getKey())); } } private static void validateModule(ProjectDefinition moduleDef, List validationMessages) { - if (!ComponentKeys.isValidModuleKey(moduleDef.getKey())) { + if (!ComponentKeys.isValidProjectKey(moduleDef.getKey())) { validationMessages.add(format("\"%s\" is not a valid project or module key. " + "Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit.", moduleDef.getKey())); } -- 2.39.5