From 7d0a0dc0fa5dd24991a57fcb947ab0f7e961508c Mon Sep 17 00:00:00 2001 From: pierre Date: Wed, 27 Oct 2021 17:45:06 +0200 Subject: [PATCH] SONAR-15574 open source background project export task processing --- .../ProjectAnalysisTaskModule.java | 4 +- .../ProjectExportComputationSteps.java | 80 ++++ .../ProjectExportContainerPopulator.java | 61 +++ .../projectexport/ProjectExportProcessor.java | 38 ++ .../analysis/ExportAnalysesStep.java | 127 ++++++ .../projectexport/analysis/package-info.java | 23 ++ .../branches/ExportBranchesStep.java | 80 ++++ .../projectexport/branches/package-info.java | 23 ++ .../component/ComponentRepository.java | 39 ++ .../component/ComponentRepositoryImpl.java | 64 +++ .../component/ExportComponentsStep.java | 126 ++++++ .../component/MutableComponentRepository.java | 31 ++ .../projectexport/component/package-info.java | 23 ++ .../file/ExportLineHashesStep.java | 147 +++++++ .../task/projectexport/file/package-info.java | 23 ++ .../issue/ExportIssuesChangelogStep.java | 116 ++++++ .../projectexport/issue/ExportIssuesStep.java | 194 +++++++++ .../projectexport/issue/package-info.java | 23 ++ .../ce/task/projectexport/package-info.java | 23 ++ .../rule/ExportAdHocRulesStep.java | 122 ++++++ .../projectexport/rule/ExportRuleStep.java | 69 ++++ .../ce/task/projectexport/rule/Rule.java | 74 ++++ .../projectexport/rule/RuleRepository.java | 40 ++ .../rule/RuleRepositoryImpl.java | 58 +++ .../task/projectexport/rule/package-info.java | 23 ++ .../task/projectexport/steps/DumpElement.java | 77 ++++ .../task/projectexport/steps/DumpReader.java | 36 ++ .../task/projectexport/steps/DumpWriter.java | 50 +++ .../projectexport/steps/DumpWriterImpl.java | 100 +++++ .../projectexport/steps/ExportEventsStep.java | 116 ++++++ .../projectexport/steps/ExportLinksStep.java | 81 ++++ .../steps/ExportLiveMeasuresStep.java | 112 +++++ .../steps/ExportMeasuresStep.java | 131 ++++++ .../steps/ExportMetricsStep.java | 75 ++++ .../steps/ExportNewCodePeriodsStep.java | 82 ++++ .../steps/ExportPluginsStep.java | 70 ++++ .../steps/ExportSettingsStep.java | 93 +++++ .../projectexport/steps/LoadProjectStep.java | 66 +++ .../projectexport/steps/MessageStream.java | 29 ++ .../steps/MessageStreamImpl.java | 65 +++ .../projectexport/steps/MetricRepository.java | 33 ++ .../steps/MutableDumpReader.java | 28 ++ .../steps/MutableDumpReaderImpl.java | 88 ++++ .../steps/MutableMetricRepository.java | 26 ++ .../steps/MutableMetricRepositoryImpl.java | 39 ++ .../steps/MutableProjectHolder.java | 31 ++ .../steps/MutableProjectHolderImpl.java | 58 +++ .../projectexport/steps/ProjectHolder.java | 31 ++ .../projectexport/steps/PublishDumpStep.java | 44 ++ .../projectexport/steps/StreamWriter.java | 29 ++ .../projectexport/steps/StreamWriterImpl.java | 54 +++ .../steps/WriteMetadataStep.java | 55 +++ .../projectexport/steps/package-info.java | 23 ++ .../taskprocessor/ProjectDescriptor.java | 85 ++++ .../ProjectExportTaskProcessor.java | 83 ++++ .../util/ProjectExportDumpFS.java | 34 ++ .../util/ProjectExportDumpFSImpl.java | 65 +++ .../util/ProjectImportDumpFS.java | 34 ++ .../util/ProjectImportDumpFSImpl.java | 66 +++ .../projectexport/util/ResultSetUtils.java | 61 +++ .../task/projectexport/util/package-info.java | 23 ++ .../java/org/sonar/ce/task/util/Files2.java | 298 +++++++++++++ .../org/sonar/ce/task/util/Protobuf2.java | 71 ++++ .../src/main/protobuf/project_dump.proto | 218 ++++++++++ .../ProjectExportComputationStepsTest.java | 73 ++++ .../ProjectExportContainerPopulatorTest.java | 105 +++++ .../analysis/ExportAnalysesStepTest.java | 223 ++++++++++ .../branches/ExportBranchesStepTest.java | 161 ++++++++ .../ComponentRepositoryImplTest.java | 119 ++++++ .../component/ExportComponentsStepTest.java | 176 ++++++++ .../file/ExportLineHashesStepTest.java | 195 +++++++++ .../issue/ExportIssuesChangelogStepTest.java | 277 +++++++++++++ .../issue/ExportIssuesStepTest.java | 391 ++++++++++++++++++ .../rule/ExportAdHocRulesStepTest.java | 232 +++++++++++ .../rule/ExportRuleStepTest.java | 123 ++++++ .../rule/RuleRepositoryImplTest.java | 139 +++++++ .../ce/task/projectexport/rule/RuleTest.java | 83 ++++ .../projectexport/steps/DumpElementTest.java | 50 +++ .../steps/DumpWriterImplTest.java | 152 +++++++ .../steps/ExportEventsStepTest.java | 170 ++++++++ .../steps/ExportLinksStepTest.java | 140 +++++++ .../steps/ExportLiveMeasuresStepTest.java | 205 +++++++++ .../steps/ExportMeasuresStepTest.java | 235 +++++++++++ .../steps/ExportMetricsStepTest.java | 129 ++++++ .../steps/ExportNewCodePeriodsStepTest.java | 178 ++++++++ .../steps/ExportPluginsStepTest.java | 103 +++++ .../steps/ExportSettingsStepTest.java | 170 ++++++++ .../projectexport/steps/FakeDumpWriter.java | 115 ++++++ .../steps/LoadProjectStepTest.java | 82 ++++ .../MutableMetricRepositoryImplTest.java | 51 +++ .../steps/PublishDumpStepTest.java | 43 ++ .../steps/WriteMetadataStepTest.java | 68 +++ .../util/ProjectExportDumpFSImplTest.java | 83 ++++ .../util/ProjectImportDumpFSImplTest.java | 83 ++++ .../org/sonar/ce/task/util/Files2Test.java | 252 +++++++++++ .../org/sonar/ce/task/util/Protobuf2Test.java | 129 ++++++ .../task/projectexport/issue/rubbish_data.txt | 3 + .../src/main/java/org/sonar/db/MyBatis.java | 2 + .../java/org/sonar/db/ce/CeTaskTypes.java | 1 + .../sonar/db/project/ProjectExportMapper.java | 38 ++ .../sonar/db/project/ProjectExportMapper.xml | 61 +++ .../ce/projectdump/ExportSubmitterImpl.java | 3 +- .../server/projectdump/ws/ExportAction.java | 9 +- .../server/projectdump/ws/example-export.json | 0 .../projectdump/ws/ExportActionTest.java | 3 +- .../platformlevel/PlatformLevel4.java | 2 + 106 files changed, 9241 insertions(+), 7 deletions(-) create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportComputationSteps.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulator.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportProcessor.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepository.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ExportComponentsStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/MutableComponentRepository.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportRuleStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/Rule.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepository.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpElement.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpReader.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriter.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriterImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportEventsStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLinksStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/LoadProjectStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStream.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStreamImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MetricRepository.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReader.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReaderImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepository.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolder.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolderImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ProjectHolder.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/PublishDumpStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriter.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriterImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectDescriptor.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectExportTaskProcessor.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFS.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFS.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ResultSetUtils.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/package-info.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Files2.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Protobuf2.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportComputationStepsTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulatorTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImplTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ExportComponentsStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportRuleStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImplTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpElementTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpWriterImplTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportEventsStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLinksStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/FakeDumpWriter.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/LoadProjectStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImplTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/PublishDumpStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImplTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImplTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Files2Test.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Protobuf2Test.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/resources/org/sonar/ce/task/projectexport/issue/rubbish_data.txt create mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java create mode 100644 server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml rename server/sonar-webserver-webapi/src/{test => main}/resources/org/sonar/server/projectdump/ws/example-export.json (100%) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/ProjectAnalysisTaskModule.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/ProjectAnalysisTaskModule.java index 82e952a5613..e6f00bfb47c 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/ProjectAnalysisTaskModule.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/ProjectAnalysisTaskModule.java @@ -21,6 +21,7 @@ package org.sonar.ce.task.projectanalysis; import org.sonar.ce.task.projectanalysis.container.ContainerFactoryImpl; import org.sonar.ce.task.projectanalysis.taskprocessor.ReportTaskProcessor; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectExportTaskProcessor; import org.sonar.ce.task.step.ComputationStepExecutor; import org.sonar.core.platform.Module; @@ -31,6 +32,7 @@ public class ProjectAnalysisTaskModule extends Module { // task ContainerFactoryImpl.class, ComputationStepExecutor.class, - ReportTaskProcessor.class); + ReportTaskProcessor.class, + ProjectExportTaskProcessor.class); } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportComputationSteps.java new file mode 100644 index 00000000000..49ceeeb9866 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportComputationSteps.java @@ -0,0 +1,80 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport; + +import java.util.Arrays; +import java.util.List; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.projectanalysis.step.AbstractComputationSteps; +import org.sonar.ce.task.projectexport.analysis.ExportAnalysesStep; +import org.sonar.ce.task.projectexport.branches.ExportBranchesStep; +import org.sonar.ce.task.projectexport.component.ExportComponentsStep; +import org.sonar.ce.task.projectexport.file.ExportLineHashesStep; +import org.sonar.ce.task.projectexport.issue.ExportIssuesChangelogStep; +import org.sonar.ce.task.projectexport.issue.ExportIssuesStep; +import org.sonar.ce.task.projectexport.rule.ExportAdHocRulesStep; +import org.sonar.ce.task.projectexport.rule.ExportRuleStep; +import org.sonar.ce.task.projectexport.steps.ExportEventsStep; +import org.sonar.ce.task.projectexport.steps.ExportLinksStep; +import org.sonar.ce.task.projectexport.steps.ExportLiveMeasuresStep; +import org.sonar.ce.task.projectexport.steps.ExportMeasuresStep; +import org.sonar.ce.task.projectexport.steps.ExportMetricsStep; +import org.sonar.ce.task.projectexport.steps.ExportNewCodePeriodsStep; +import org.sonar.ce.task.projectexport.steps.ExportPluginsStep; +import org.sonar.ce.task.projectexport.steps.ExportSettingsStep; +import org.sonar.ce.task.projectexport.steps.LoadProjectStep; +import org.sonar.ce.task.projectexport.steps.PublishDumpStep; +import org.sonar.ce.task.projectexport.steps.WriteMetadataStep; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.ce.task.step.ExecuteStatelessInitExtensionsStep; + +public class ProjectExportComputationSteps extends AbstractComputationSteps { + private static final List> STEPS_CLASSES = Arrays.asList( + LoadProjectStep.class, + ExecuteStatelessInitExtensionsStep.class, + WriteMetadataStep.class, + ExportComponentsStep.class, + ExportSettingsStep.class, + ExportPluginsStep.class, + ExportBranchesStep.class, + ExportAnalysesStep.class, + ExportMeasuresStep.class, + ExportLiveMeasuresStep.class, + ExportMetricsStep.class, + ExportIssuesStep.class, + ExportIssuesChangelogStep.class, + ExportRuleStep.class, + ExportAdHocRulesStep.class, + ExportLinksStep.class, + ExportEventsStep.class, + ExportLineHashesStep.class, + ExportNewCodePeriodsStep.class, + PublishDumpStep.class); + + public ProjectExportComputationSteps(TaskContainer container) { + super(container); + } + + @Override + public List> orderedStepClasses() { + return STEPS_CLASSES; + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulator.java new file mode 100644 index 00000000000..7ffa25e6871 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulator.java @@ -0,0 +1,61 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport; + +import java.util.List; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.projectexport.rule.RuleRepositoryImpl; +import org.sonar.ce.task.projectexport.steps.DumpWriterImpl; +import org.sonar.ce.task.projectexport.steps.MutableMetricRepositoryImpl; +import org.sonar.ce.task.projectexport.steps.MutableProjectHolderImpl; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.projectexport.util.ProjectExportDumpFSImpl; +import org.sonar.ce.task.setting.SettingsLoader; +import org.sonar.core.platform.ContainerPopulator; + +public class ProjectExportContainerPopulator implements ContainerPopulator { + private static final List> COMPONENT_CLASSES = List.of( + SettingsLoader.class, + ProjectExportProcessor.class, + MutableMetricRepositoryImpl.class, + MutableProjectHolderImpl.class, + ProjectExportDumpFSImpl.class, + DumpWriterImpl.class, + RuleRepositoryImpl.class, + ComponentRepositoryImpl.class); + + private final ProjectDescriptor projectDescriptor; + + public ProjectExportContainerPopulator(ProjectDescriptor descriptor) { + this.projectDescriptor = descriptor; + } + + @Override + public void populateContainer(TaskContainer container) { + ProjectExportComputationSteps steps = new ProjectExportComputationSteps(container); + + container.add(projectDescriptor); + container.add(steps); + container.addSingletons(COMPONENT_CLASSES); + container.addSingletons(steps.orderedStepClasses()); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportProcessor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportProcessor.java new file mode 100644 index 00000000000..a7366345b65 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/ProjectExportProcessor.java @@ -0,0 +1,38 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport; + +import org.sonar.ce.task.CeTaskInterrupter; +import org.sonar.ce.task.step.ComputationStepExecutor; + +public class ProjectExportProcessor { + private final ProjectExportComputationSteps steps; + private final CeTaskInterrupter taskInterrupter; + + public ProjectExportProcessor(ProjectExportComputationSteps steps, CeTaskInterrupter taskInterrupter) { + this.steps = steps; + this.taskInterrupter = taskInterrupter; + } + + public void process() { + new ComputationStepExecutor(this.steps, taskInterrupter).execute(); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStep.java new file mode 100644 index 00000000000..be16ab61618 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStep.java @@ -0,0 +1,127 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.analysis; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; +import static org.sonar.db.DatabaseUtils.getString; +import static org.sonar.db.component.SnapshotDto.STATUS_PROCESSED; + +public class ExportAnalysesStep implements ComputationStep { + + // contrary to naming convention used in other tables, the column snapshots.created_at is + // the functional date (for instance when using property sonar.projectDate=2010-01-01). The + // column "build_date" is the technical date of scanner execution. It must not be exported. + private static final String QUERY = "select" + + " p.uuid, s.version, s.created_at," + + " s.period1_mode, s.period1_param, s.period1_date," + + " s.uuid, s.build_string" + + " from snapshots s" + + " inner join components p on s.component_uuid=p.uuid" + + " inner join project_branches pb on pb.uuid=p.uuid" + + " where pb.project_uuid=? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge=? and s.status=? and p.enabled=?" + + " order by s.build_date asc"; + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final ComponentRepository componentRepository; + private final DumpWriter dumpWriter; + + public ExportAnalysesStep(DbClient dbClient, ProjectHolder projectHolder, ComponentRepository componentRepository, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.componentRepository = componentRepository; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.ANALYSES); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = buildSelectStatement(dbSession); + ResultSet rs = stmt.executeQuery()) { + + ProjectDump.Analysis.Builder builder = ProjectDump.Analysis.newBuilder(); + while (rs.next()) { + // Results are ordered by ascending id so that any parent is located + // before its children. + ProjectDump.Analysis analysis = convertToAnalysis(rs, builder); + output.write(analysis); + count++; + } + Loggers.get(getClass()).debug("{} analyses exported", count); + + } catch (Exception e) { + throw new IllegalStateException(format("Analysis Export failed after processing %d analyses successfully", count), e); + } + } + + private PreparedStatement buildSelectStatement(DbSession dbSession) throws SQLException { + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY); + try { + stmt.setString(1, projectHolder.projectDto().getUuid()); + stmt.setBoolean(2, true); + stmt.setString(3, STATUS_PROCESSED); + stmt.setBoolean(4, true); + return stmt; + } catch (Exception t) { + DatabaseUtils.closeQuietly(stmt); + throw t; + } + } + + private ProjectDump.Analysis convertToAnalysis(ResultSet rs, ProjectDump.Analysis.Builder builder) throws SQLException { + builder.clear(); + long componentRef = componentRepository.getRef(rs.getString(1)); + return builder + .setComponentRef(componentRef) + .setProjectVersion(defaultString(getString(rs, 2))) + .setDate(rs.getLong(3)) + .setPeriod1Mode(defaultString(getString(rs, 4))) + .setPeriod1Param(defaultString(getString(rs, 5))) + .setPeriod1Date(rs.getLong(6)) + .setUuid(rs.getString(7)) + .setBuildString(defaultString(getString(rs, 8))) + .build(); + } + + @Override + public String getDescription() { + return "Export analyses"; + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/package-info.java new file mode 100644 index 00000000000..fa89cf568e3 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/analysis/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.analysis; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStep.java new file mode 100644 index 00000000000..1f393ccff85 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStep.java @@ -0,0 +1,80 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.branches; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.BranchDto; +import org.sonar.db.project.ProjectExportMapper; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; + +public class ExportBranchesStep implements ComputationStep { + + private final DumpWriter dumpWriter; + private final DbClient dbClient; + private final ProjectHolder projectHolder; + + public ExportBranchesStep(DumpWriter dumpWriter, DbClient dbClient, ProjectHolder projectHolder) { + this.dumpWriter = dumpWriter; + this.dbClient = dbClient; + this.projectHolder = projectHolder; + } + + @Override + public void execute(Context context) { + long count = 0L; + try { + try (DbSession dbSession = dbClient.openSession(false); + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.BRANCHES)) { + ProjectDump.Branch.Builder builder = ProjectDump.Branch.newBuilder(); + List branches = dbSession.getMapper(ProjectExportMapper.class).selectBranchesForExport(projectHolder.projectDto().getUuid()); + for (BranchDto branch : branches) { + builder + .clear() + .setUuid(branch.getUuid()) + .setProjectUuid(branch.getProjectUuid()) + .setKee(branch.getKey()) + .setBranchType(branch.getBranchType().name()) + .setMergeBranchUuid(defaultString(branch.getMergeBranchUuid())); + output.write(builder.build()); + ++count; + } + Loggers.get(getClass()).debug("{} branches exported", count); + } + } catch (Exception e) { + throw new IllegalStateException(format("Branch export failed after processing %d branch(es) successfully", count), e); + } + } + + @Override + public String getDescription() { + return "Export branches"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/package-info.java new file mode 100644 index 00000000000..b5740467269 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/branches/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.branches; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepository.java new file mode 100644 index 00000000000..a3698675f99 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepository.java @@ -0,0 +1,39 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.component; + +import java.util.Set; + +/** + * Holds the references of components which are present in the dump. + */ +public interface ComponentRepository { + + /** + * @throws IllegalStateException if there is no ref for the specified Uuid in the repository + */ + long getRef(String uuid); + + /** + * Uuids of the components of type FILE (ie. Qualifiers = {@link org.sonar.api.resources.Qualifiers#FILE}) known to + * the repository. + */ + Set getFileUuids(); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImpl.java new file mode 100644 index 00000000000..5bcb5a0ac9a --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImpl.java @@ -0,0 +1,64 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.component; + +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class ComponentRepositoryImpl implements MutableComponentRepository { + private final Map refsByUuid = new HashMap<>(); + private final Set fileUuids = new HashSet<>(); + + @Override + public void register(long ref, String uuid, boolean file) { + requireNonNull(uuid, "uuid can not be null"); + Long existingRef = refsByUuid.get(uuid); + if (existingRef != null) { + checkArgument(ref == existingRef, "Uuid '%s' already registered under ref '%s' in repository", uuid, existingRef); + boolean existingIsFile = fileUuids.contains(uuid); + checkArgument(file == existingIsFile, "Uuid '%s' already registered but %sas a File", uuid, existingIsFile ? "" : "not "); + } else { + refsByUuid.put(uuid, ref); + if (file) { + fileUuids.add(uuid); + } + } + } + + @Override + public long getRef(String uuid) { + Long ref = refsByUuid.get(requireNonNull(uuid, "uuid can not be null")); + checkState(ref != null, "No reference registered in the repository for uuid '%s'", uuid); + + return ref; + } + + @Override + public Set getFileUuids() { + return ImmutableSet.copyOf(fileUuids); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ExportComponentsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ExportComponentsStep.java new file mode 100644 index 00000000000..629de0bf1cb --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/ExportComponentsStep.java @@ -0,0 +1,126 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.component; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; +import static org.sonar.db.DatabaseUtils.getString; + +public class ExportComponentsStep implements ComputationStep { + + // Results are ordered by ascending id so that any parent is located + // before its children. + private static final String QUERY = "select" + + " p.uuid, p.qualifier, p.uuid_path, p.kee, p.name," + + " p.description, p.scope, p.language, p.long_name, p.path," + + " p.module_uuid, p.module_uuid_path, p.deprecated_kee, p.project_uuid, p.main_branch_project_uuid" + + " from components p" + + " join components pp on pp.uuid = p.project_uuid" + + " join project_branches pb on pb.uuid = pp.uuid" + + " where pb.project_uuid=? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge=? and p.enabled=?"; + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final MutableComponentRepository componentRepository; + private final DumpWriter dumpWriter; + + public ExportComponentsStep(DbClient dbClient, ProjectHolder projectHolder, MutableComponentRepository componentRepository, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.componentRepository = componentRepository; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long ref = 1; + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.COMPONENTS); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = createSelectStatement(dbSession); + ResultSet rs = stmt.executeQuery()) { + ProjectDump.Component.Builder componentBuilder = ProjectDump.Component.newBuilder(); + while (rs.next()) { + String uuid = getString(rs, 1); + String qualifier = getString(rs, 2); + String uuidPath = getString(rs, 3); + componentBuilder.clear(); + componentRepository.register(ref, uuid, Qualifiers.FILE.equals(qualifier)); + ProjectDump.Component component = componentBuilder + .setRef(ref) + .setUuid(uuid) + .setUuidPath(uuidPath) + .setKey(getString(rs, 4)) + .setName(defaultString(getString(rs, 5))) + .setDescription(defaultString(getString(rs, 6))) + .setScope(getString(rs, 7)) + .setQualifier(qualifier) + .setLanguage(defaultString(getString(rs, 8))) + .setLongName(defaultString(getString(rs, 9))) + .setPath(defaultString(getString(rs, 10))) + .setModuleUuid(defaultString(getString(rs, 11))) + .setModuleUuidPath(defaultString(getString(rs, 12))) + .setDeprecatedKey(defaultString(getString(rs, 13))) + .setProjectUuid(getString(rs, 14)) + .setMainBranchProjectUuid(defaultString(getString(rs, 15))) + .build(); + output.write(component); + ref++; + count++; + } + Loggers.get(getClass()).debug("{} components exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Component Export failed after processing %d components successfully", count), e); + } + } + + private PreparedStatement createSelectStatement(DbSession dbSession) throws SQLException { + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY); + try { + stmt.setString(1, projectHolder.projectDto().getUuid()); + stmt.setBoolean(2, true); + stmt.setBoolean(3, true); + return stmt; + } catch (Exception t) { + DatabaseUtils.closeQuietly(stmt); + throw t; + } + } + + @Override + public String getDescription() { + return "Export components"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/MutableComponentRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/MutableComponentRepository.java new file mode 100644 index 00000000000..709d468e448 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/MutableComponentRepository.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.component; + +public interface MutableComponentRepository extends ComponentRepository { + /** + * Adds the reference of a Component (designated by it's uuid) to the repository and keep tracks of whether + * specified uuid is one of a File. + * + * @throws IllegalArgumentException if the specified uuid is already registered with another ref in the repository. + * @throws IllegalArgumentException if the specified uuid is already registered with another File flag + */ + void register(long ref, String uuid, boolean file); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/package-info.java new file mode 100644 index 00000000000..059e4a44f99 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/component/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.component; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStep.java new file mode 100644 index 00000000000..939a07bdeaf --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStep.java @@ -0,0 +1,147 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.file; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Set; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; + +import static com.google.common.collect.Iterables.partition; +import static java.lang.String.format; +import static org.sonar.ce.task.projectexport.util.ResultSetUtils.emptyIfNull; +import static org.sonar.db.DatabaseUtils.PARTITION_SIZE_FOR_ORACLE; +import static org.sonar.db.DatabaseUtils.closeQuietly; + +public class ExportLineHashesStep implements ComputationStep { + + private final DbClient dbClient; + private final DumpWriter dumpWriter; + private final ComponentRepository componentRepository; + + public ExportLineHashesStep(DbClient dbClient, DumpWriter dumpWriter, ComponentRepository componentRepository) { + this.dbClient = dbClient; + this.dumpWriter = dumpWriter; + this.componentRepository = componentRepository; + } + + @Override + public String getDescription() { + return "Export line hashes"; + } + + @Override + public void execute(Context context) { + Set allFileUuids = componentRepository.getFileUuids(); + PreparedStatement stmt = null; + ResultSet rs = null; + long count = 0; + try (StreamWriter output = dumpWriter.newStreamWriter(DumpElement.LINES_HASHES)) { + if (allFileUuids.isEmpty()) { + return; + } + try (DbSession dbSession = dbClient.openSession(false)) { + ProjectDump.LineHashes.Builder builder = ProjectDump.LineHashes.newBuilder(); + for (List fileUuids : partition(allFileUuids, PARTITION_SIZE_FOR_ORACLE)) { + stmt = createStatement(dbSession, fileUuids); + rs = stmt.executeQuery(); + while (rs.next()) { + ProjectDump.LineHashes lineHashes = toLinehashes(builder, rs); + output.write(lineHashes); + count++; + } + closeQuietly(rs); + closeQuietly(stmt); + } + Loggers.get(getClass()).debug("Lines hashes of {} files exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Lines hashes export failed after processing %d files successfully", count), e); + } finally { + closeQuietly(rs); + closeQuietly(stmt); + } + } + } + + private PreparedStatement createStatement(DbSession dbSession, List uuids) throws SQLException { + String sql = "select" + + " file_uuid, line_hashes, project_uuid" + + " FROM file_sources" + + " WHERE file_uuid in (%s)" + + " order by created_at, uuid"; + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, format(sql, wildCardStringFor(uuids))); + try { + int i = 1; + for (String uuid : uuids) { + stmt.setString(i, uuid); + i++; + } + return stmt; + } catch (Exception e) { + DatabaseUtils.closeQuietly(stmt); + throw e; + } + } + + private static String wildCardStringFor(List uuids) { + switch (uuids.size()) { + case 0: + throw new IllegalArgumentException("uuids can not be empty"); + case 1: + return "?"; + default: + return createWildCardStringFor(uuids); + } + } + + private static String createWildCardStringFor(List uuids) { + int size = (uuids.size() * 2) - 1; + char[] res = new char[size]; + for (int j = 0; j < size; j++) { + if (j % 2 == 0) { + res[j] = '?'; + } else { + res[j] = ','; + } + } + return new String(res); + } + + private ProjectDump.LineHashes toLinehashes(ProjectDump.LineHashes.Builder builder, ResultSet rs) throws SQLException { + builder.clear(); + + return builder + .setComponentRef(componentRepository.getRef(rs.getString(1))) + .setHashes(emptyIfNull(rs, 2)) + .setProjectUuid(emptyIfNull(rs, 3)) + .build(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/package-info.java new file mode 100644 index 00000000000..a9ed337518e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/file/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.file; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStep.java new file mode 100644 index 00000000000..2153ef39236 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStep.java @@ -0,0 +1,116 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.issue; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; + +import static java.lang.String.format; +import static org.sonar.ce.task.projectexport.util.ResultSetUtils.defaultIfNull; +import static org.sonar.ce.task.projectexport.util.ResultSetUtils.emptyIfNull; + +public class ExportIssuesChangelogStep implements ComputationStep { + private static final String STATUS_CLOSED = "CLOSED"; + private static final String QUERY = "select" + + " ic.kee, ic.issue_key, ic.change_type, ic.change_data, ic.user_login," + + " ic.issue_change_creation_date, ic.created_at, p.uuid" + + " from issue_changes ic" + + " join issues i on i.kee = ic.issue_key" + + " join projects p on p.uuid = i.project_uuid" + + " join project_branches pb on pb.uuid = p.uuid" + + " where pb.project_uuid = ? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge = ? " + + " and i.status <> ?" + + " order by ic.created_at asc"; + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final DumpWriter dumpWriter; + + public ExportIssuesChangelogStep(DbClient dbClient, ProjectHolder projectHolder, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.dumpWriter = dumpWriter; + } + + @Override + public String getDescription() { + return "Export issues changelog"; + } + + @Override + public void execute(Context context) { + long count = 0; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.ISSUES_CHANGELOG); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = createStatement(dbSession); + ResultSet rs = stmt.executeQuery()) { + ProjectDump.IssueChange.Builder builder = ProjectDump.IssueChange.newBuilder(); + while (rs.next()) { + ProjectDump.IssueChange issue = toIssuesChange(builder, rs); + output.write(issue); + count++; + } + Loggers.get(getClass()).debug("{} issue changes exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Issues changelog export failed after processing %d issue changes successfully", count), e); + } + } + + private PreparedStatement createStatement(DbSession dbSession) throws SQLException { + // export oldest entries first, so they can be imported with smallest ids + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY); + try { + stmt.setString(1, projectHolder.projectDto().getUuid()); + stmt.setBoolean(2, true); + stmt.setString(3, STATUS_CLOSED); + return stmt; + } catch (Exception t) { + DatabaseUtils.closeQuietly(stmt); + throw t; + } + } + + private static ProjectDump.IssueChange toIssuesChange(ProjectDump.IssueChange.Builder builder, ResultSet rs) throws SQLException { + builder.clear(); + + return builder + .setKey(emptyIfNull(rs, 1)) + .setIssueUuid(rs.getString(2)) + .setChangeType(emptyIfNull(rs, 3)) + .setChangeData(emptyIfNull(rs, 4)) + .setUserUuid(emptyIfNull(rs, 5)) + .setCreatedAt(defaultIfNull(rs, 6, defaultIfNull(rs, 7, 0L))) + .setProjectUuid(rs.getString(8)) + .build(); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java new file mode 100644 index 00000000000..3d2b97c5f83 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java @@ -0,0 +1,194 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.issue; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Objects; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.projectexport.rule.Rule; +import org.sonar.ce.task.projectexport.rule.RuleRepository; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.protobuf.DbIssues; + +import static java.lang.String.format; +import static org.sonar.ce.task.projectexport.util.ResultSetUtils.defaultIfNull; +import static org.sonar.ce.task.projectexport.util.ResultSetUtils.emptyIfNull; + +public class ExportIssuesStep implements ComputationStep { + private static final String RULE_STATUS_REMOVED = "REMOVED"; + private static final String ISSUE_STATUS_CLOSED = "CLOSED"; + + // ordered by rule_id to reduce calls to RuleRepository + private static final String QUERY = "select" + + " i.kee, r.uuid, r.plugin_rule_key, r.plugin_name, i.issue_type," + + " i.component_uuid, i.message, i.line, i.checksum, i.status," + + " i.resolution, i.severity, i.manual_severity, i.gap, effort," + + " i.assignee, i.author_login, i.tags, i.issue_attributes, i.issue_creation_date," + + " i.issue_update_date, i.issue_close_date, i.locations, i.project_uuid" + + " from issues i" + + " join rules r on r.uuid = i.rule_uuid and r.status <> ?" + + " join components p on p.uuid = i.project_uuid" + + " join project_branches pb on pb.uuid = p.uuid" + + " where pb.project_uuid = ? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge=?" + + " and i.status <> ?" + + " order by" + + " i.rule_uuid asc, i.created_at asc"; + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final DumpWriter dumpWriter; + private final RuleRegistrar ruleRegistrar; + private final ComponentRepository componentRepository; + + public ExportIssuesStep(DbClient dbClient, ProjectHolder projectHolder, DumpWriter dumpWriter, RuleRepository ruleRepository, + ComponentRepository componentRepository) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.dumpWriter = dumpWriter; + this.componentRepository = componentRepository; + this.ruleRegistrar = new RuleRegistrar(ruleRepository); + } + + @Override + public String getDescription() { + return "Export issues"; + } + + @Override + public void execute(Context context) { + long count = 0; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.ISSUES); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = createStatement(dbSession); + ResultSet rs = stmt.executeQuery()) { + ProjectDump.Issue.Builder builder = ProjectDump.Issue.newBuilder(); + while (rs.next()) { + ProjectDump.Issue issue = toIssue(builder, rs); + output.write(issue); + count++; + } + Loggers.get(getClass()).debug("{} issues exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Issue export failed after processing %d issues successfully", count), e); + } + } + + private PreparedStatement createStatement(DbSession dbSession) throws SQLException { + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY); + try { + stmt.setString(1, RULE_STATUS_REMOVED); + stmt.setString(2, projectHolder.projectDto().getUuid()); + stmt.setBoolean(3, true); + stmt.setString(4, ISSUE_STATUS_CLOSED); + return stmt; + } catch (Exception t) { + DatabaseUtils.closeQuietly(stmt); + throw t; + } + } + + private ProjectDump.Issue toIssue(ProjectDump.Issue.Builder builder, ResultSet rs) throws SQLException { + builder.clear(); + String issueUuid = rs.getString(1); + setRule(builder, rs); + builder + .setUuid(issueUuid) + .setType(rs.getInt(5)) + .setComponentRef(componentRepository.getRef(rs.getString(6))) + .setMessage(emptyIfNull(rs, 7)) + .setLine(rs.getInt(8)) + .setChecksum(emptyIfNull(rs, 9)) + .setStatus(emptyIfNull(rs, 10)) + .setResolution(emptyIfNull(rs, 11)) + .setSeverity(emptyIfNull(rs, 12)) + .setManualSeverity(rs.getBoolean(13)) + .setGap(defaultIfNull(rs, 14, DumpElement.ISSUES.NO_GAP)) + .setEffort(defaultIfNull(rs, 15, DumpElement.ISSUES.NO_EFFORT)) + .setAssignee(emptyIfNull(rs, 16)) + .setAuthor(emptyIfNull(rs, 17)) + .setTags(emptyIfNull(rs, 18)) + .setAttributes(emptyIfNull(rs, 19)) + .setIssueCreatedAt(rs.getLong(20)) + .setIssueUpdatedAt(rs.getLong(21)) + .setIssueClosedAt(rs.getLong(22)) + .setProjectUuid(rs.getString(24)); + setLocations(builder, rs, issueUuid); + return builder.build(); + } + + private void setRule(ProjectDump.Issue.Builder builder, ResultSet rs) throws SQLException { + String ruleUuid = rs.getString(2); + String ruleKey = rs.getString(3); + String repositoryKey = rs.getString(4); + builder.setRuleRef(ruleRegistrar.register(ruleUuid, repositoryKey, ruleKey).getRef()); + } + + private static void setLocations(ProjectDump.Issue.Builder builder, ResultSet rs, String issueUuid) throws SQLException { + try { + byte[] bytes = rs.getBytes(23); + if (bytes != null) { + // fail fast, ensure we can read data from DB + DbIssues.Locations.parseFrom(bytes); + builder.setLocations(ByteString.copyFrom(bytes)); + } + } catch (InvalidProtocolBufferException e) { + throw new IllegalStateException(format("Fail to read locations from DB for issue %s", issueUuid), e); + } + } + + private static class RuleRegistrar { + private final RuleRepository ruleRepository; + private Rule previousRule = null; + private String previousRuleUuid = null; + + private RuleRegistrar(RuleRepository ruleRepository) { + this.ruleRepository = ruleRepository; + } + + public Rule register(String ruleUuid, String repositoryKey, String ruleKey) { + if (Objects.equals(previousRuleUuid, ruleUuid)) { + return previousRule; + } + return lookup(ruleUuid, RuleKey.of(repositoryKey, ruleKey)); + } + + private Rule lookup(String ruleUuid, RuleKey ruleKey) { + this.previousRule = ruleRepository.register(ruleUuid, ruleKey); + this.previousRuleUuid = ruleUuid; + return previousRule; + } + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/package-info.java new file mode 100644 index 00000000000..81ff124b18a --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.issue; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/package-info.java new file mode 100644 index 00000000000..0c6ae7f0a59 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStep.java new file mode 100644 index 00000000000..31f7665d9f7 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStep.java @@ -0,0 +1,122 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; + +import static java.lang.String.format; +import static org.sonar.ce.task.projectexport.util.ResultSetUtils.defaultIfNull; +import static org.sonar.ce.task.projectexport.util.ResultSetUtils.emptyIfNull; + +public class ExportAdHocRulesStep implements ComputationStep { + private static final String RULE_STATUS_REMOVED = "REMOVED"; + private static final String ISSUE_STATUS_CLOSED = "CLOSED"; + + private static final String QUERY = "select" + + " r.uuid, r.plugin_key, r.plugin_rule_key, r.plugin_name, r.name, r.status, r.rule_type, r.scope, rm.rule_uuid, rm.ad_hoc_name," + + " rm.ad_hoc_description,rm.ad_hoc_severity, rm.ad_hoc_type" + + " from rules r" + + " left join rules_metadata rm on rm.rule_uuid = r.uuid" + + " inner join issues i on r.uuid = i.rule_uuid and r.status <> ? and r.is_ad_hoc = ?" + + " left join components p on p.uuid = i.project_uuid" + + " left join project_branches pb on pb.uuid = p.uuid" + + " where pb.project_uuid = ? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge = ?" + + " and i.status <> ?" + + " order by" + + " i.rule_uuid asc"; + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final DumpWriter dumpWriter; + + public ExportAdHocRulesStep(DbClient dbClient, ProjectHolder projectHolder, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.AD_HOC_RULES); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY)) { + stmt.setString(1, RULE_STATUS_REMOVED); + stmt.setBoolean(2, true); + stmt.setString(3, projectHolder.projectDto().getUuid()); + stmt.setBoolean(4, true); + stmt.setString(5, ISSUE_STATUS_CLOSED); + try (ResultSet rs = stmt.executeQuery()) { + ProjectDump.AdHocRule.Builder adHocRuleBuilder = ProjectDump.AdHocRule.newBuilder(); + while (rs.next()) { + ProjectDump.AdHocRule rule = convertToAdHocRule(rs, adHocRuleBuilder); + output.write(rule); + count++; + } + Loggers.get(getClass()).debug("{} ad-hoc rules exported", count); + } + } catch (Exception e) { + throw new IllegalStateException(format("Ad-hoc rules export failed after processing %d rules successfully", count), e); + } + } + + private static ProjectDump.AdHocRule convertToAdHocRule(ResultSet rs, ProjectDump.AdHocRule.Builder builder) throws SQLException { + rs.getString(9); + boolean metadataExistsForCurrentRule = !rs.wasNull(); + + builder + .clear() + .setRef(rs.getString(1)) + .setPluginKey(emptyIfNull(rs, 2)) + .setPluginRuleKey(rs.getString(3)) + .setPluginName(rs.getString(4)) + .setName(emptyIfNull(rs, 5)) + .setStatus(emptyIfNull(rs, 6)) + .setType(rs.getInt(7)) + .setScope(rs.getString(8)); + if (metadataExistsForCurrentRule) { + builder.setMetadata(ProjectDump.AdHocRule.RuleMetadata.newBuilder() + .setAdHocName(emptyIfNull(rs, 10)) + .setAdHocDescription(emptyIfNull(rs, 11)) + .setAdHocSeverity(emptyIfNull(rs, 12)) + .setAdHocType(defaultIfNull(rs, 13, 0)) + .build()); + } + + return builder.build(); + } + + @Override + public String getDescription() { + return "Export ad-hoc rules"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportRuleStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportRuleStep.java new file mode 100644 index 00000000000..c07ca05618c --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/ExportRuleStep.java @@ -0,0 +1,69 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.DumpWriter; +import org.sonar.ce.task.projectexport.steps.StreamWriter; +import org.sonar.ce.task.step.ComputationStep; + +import static java.lang.String.format; + +public class ExportRuleStep implements ComputationStep { + private final RuleRepository ruleRepository; + private final DumpWriter dumpWriter; + + public ExportRuleStep(RuleRepository ruleRepository, DumpWriter dumpWriter) { + this.ruleRepository = ruleRepository; + this.dumpWriter = dumpWriter; + } + + @Override + public String getDescription() { + return "Export rules"; + } + + @Override + public void execute(Context context) { + long count = 0; + try (StreamWriter writer = dumpWriter.newStreamWriter(DumpElement.RULES)) { + ProjectDump.Rule.Builder ruleBuilder = ProjectDump.Rule.newBuilder(); + for (Rule rule : ruleRepository.getAll()) { + ProjectDump.Rule ruleMessage = toRuleMessage(ruleBuilder, rule); + writer.write(ruleMessage); + count++; + } + Loggers.get(getClass()).debug("{} rules exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Rule Export failed after processing %d rules successfully", count), e); + } + } + + private static ProjectDump.Rule toRuleMessage(ProjectDump.Rule.Builder ruleBuilder, Rule rule) { + ruleBuilder.clear(); + return ruleBuilder + .setRef(rule.getRef()) + .setKey(rule.getKey()) + .setRepository(rule.getRepository()) + .build(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/Rule.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/Rule.java new file mode 100644 index 00000000000..b3b05c961a7 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/Rule.java @@ -0,0 +1,74 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import java.util.Objects; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import static java.util.Objects.requireNonNull; + +@Immutable +public final class Rule { + private final String ref; + private final String repository; + private final String key; + + public Rule(String ref, String repository, String key) { + this.ref = ref; + this.repository = requireNonNull(repository, "repository can not be null"); + this.key = requireNonNull(key, "key can not be null"); + } + + public String getRef() { + return ref; + } + + public String getRepository() { + return repository; + } + + public String getKey() { + return key; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Rule)) { + return false; + } + Rule rule = (Rule) o; + return repository.equals(rule.repository) && key.equals(rule.key); + } + + @Override + public int hashCode() { + return Objects.hash(repository, key); + } + + @Override + public String toString() { + return "Rule{" + + "ref='" + ref + "', repository='" + repository + "', key='" + key + "'}"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepository.java new file mode 100644 index 00000000000..f57cf9b8724 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepository.java @@ -0,0 +1,40 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import java.util.Collection; +import org.sonar.api.rule.RuleKey; + +/** + * This repository is responsible to register ids for {@link RuleKey}s and keeping track of them so that it can return + * all of them in {@link #getAll()}. + */ +public interface RuleRepository { + + /** + * Register the specified ref for the specified ruleKey and return it's representing {@link Rule} object. + * + * @throws IllegalArgumentException if the specified ruleKey is not the same as the one already in the repository (if any) + */ + Rule register(String ref, RuleKey ruleKey); + + Collection getAll(); + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImpl.java new file mode 100644 index 00000000000..736c8cff6bd --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImpl.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import com.google.common.collect.ImmutableList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.sonar.api.rule.RuleKey; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class RuleRepositoryImpl implements RuleRepository { + private final Map rulesByUuid = new HashMap<>(); + + @Override + public Rule register(String ref, RuleKey ruleKey) { + requireNonNull(ruleKey, "ruleKey can not be null"); + + Rule rule = rulesByUuid.get(ref); + if (rule != null) { + if (!ruleKey.repository().equals(rule.getRepository()) || !ruleKey.rule().equals(rule.getKey())) { + throw new IllegalArgumentException(format( + "Specified RuleKey '%s' is not equal to the one already registered in repository for ref %s: '%s'", + ruleKey, ref, RuleKey.of(rule.getRepository(), rule.getKey()))); + } + return rule; + } + + rule = new Rule(ref, ruleKey.repository(), ruleKey.rule()); + rulesByUuid.put(ref, rule); + return rule; + } + + @Override + public Collection getAll() { + return ImmutableList.copyOf(rulesByUuid.values()); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/package-info.java new file mode 100644 index 00000000000..f0d150266a9 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/rule/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.rule; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpElement.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpElement.java new file mode 100644 index 00000000000..708fc374688 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpElement.java @@ -0,0 +1,77 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; +import com.google.protobuf.Parser; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import javax.annotation.concurrent.Immutable; + +/** + * File element in the dump report. + */ +@Immutable +public class DumpElement { + public static final long NO_DATETIME = 0; + + public static final DumpElement METADATA = new DumpElement<>("metadata.pb", ProjectDump.Metadata.parser()); + public static final DumpElement COMPONENTS = new DumpElement<>("components.pb", ProjectDump.Component.parser()); + public static final DumpElement BRANCHES = new DumpElement<>("branches.pb", ProjectDump.Branch.parser()); + public static final DumpElement ANALYSES = new DumpElement<>("analyses.pb", ProjectDump.Analysis.parser()); + public static final DumpElement MEASURES = new DumpElement<>("measures.pb", ProjectDump.Measure.parser()); + public static final DumpElement LIVE_MEASURES = new DumpElement<>("live_measures.pb", ProjectDump.LiveMeasure.parser()); + public static final DumpElement METRICS = new DumpElement<>("metrics.pb", ProjectDump.Metric.parser()); + public static final IssueDumpElement ISSUES = new IssueDumpElement(); + public static final DumpElement ISSUES_CHANGELOG = new DumpElement<>("issues_changelog.pb", ProjectDump.IssueChange.parser()); + public static final DumpElement AD_HOC_RULES = new DumpElement<>("ad_hoc_rules.pb", ProjectDump.AdHocRule.parser()); + public static final DumpElement RULES = new DumpElement<>("rules.pb", ProjectDump.Rule.parser()); + public static final DumpElement LINKS = new DumpElement<>("links.pb", ProjectDump.Link.parser()); + public static final DumpElement EVENTS = new DumpElement<>("events.pb", ProjectDump.Event.parser()); + public static final DumpElement SETTINGS = new DumpElement<>("settings.pb", ProjectDump.Setting.parser()); + public static final DumpElement PLUGINS = new DumpElement<>("plugins.pb", ProjectDump.Plugin.parser()); + public static final DumpElement LINES_HASHES = new DumpElement<>("lines_hashes.pb", ProjectDump.LineHashes.parser()); + public static final DumpElement NEW_CODE_PERIODS = new DumpElement<>("new_code_periods.pb", ProjectDump.NewCodePeriod.parser()); + + private final String filename; + private final Parser parser; + + private DumpElement(String filename, Parser parser) { + this.filename = filename; + this.parser = parser; + } + + public String filename() { + return filename; + } + + public Parser parser() { + return parser; + } + + public static class IssueDumpElement extends DumpElement { + public final int NO_LINE = 0; + public final double NO_GAP = -1; + public final long NO_EFFORT = -1; + + public IssueDumpElement() { + super("issues.pb", ProjectDump.Issue.parser()); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpReader.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpReader.java new file mode 100644 index 00000000000..ba75eee1972 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpReader.java @@ -0,0 +1,36 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.io.File; + +public interface DumpReader { + + File zipFile(); + + File tempRootDir(); + + ProjectDump.Metadata metadata(); + + MessageStream stream(DumpElement elt); + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriter.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriter.java new file mode 100644 index 00000000000..6bfae1231ec --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriter.java @@ -0,0 +1,50 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; + +/** + * Writes project dump on disk + */ +public interface DumpWriter { + + /** + * Writes a single metadata message. Can be called only once. + * @throws IllegalStateException if metadata has already been written + * @throws IllegalStateException if already published (see {@link #publish()}) + */ + void write(ProjectDump.Metadata metadata); + + /** + * Writes a stream of protobuf objects. Streams are appended. + * @throws IllegalStateException if already published (see {@link #publish()}) + */ + StreamWriter newStreamWriter(DumpElement elt); + + /** + * Publishes the dump file by zipping directory and moving zip file to directory /data. + * @throws IllegalStateException if metadata has not been written + * @throws IllegalStateException if already published + */ + void publish(); + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriterImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriterImpl.java new file mode 100644 index 00000000000..3905550a96d --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/DumpWriterImpl.java @@ -0,0 +1,100 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import org.sonar.api.utils.TempFolder; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.projectexport.util.ProjectExportDumpFS; + +import static org.apache.commons.io.FileUtils.sizeOf; +import static org.sonar.ce.task.projectexport.steps.DumpElement.METADATA; +import static org.sonar.ce.task.util.Files2.FILES2; +import static org.sonar.ce.task.util.Protobuf2.PROTOBUF2; +import static org.sonar.core.util.FileUtils.humanReadableByteCountSI; + +public class DumpWriterImpl implements DumpWriter { + + private final ProjectDescriptor descriptor; + private final ProjectExportDumpFS projectExportDumpFS; + private final TempFolder tempFolder; + private final File rootDir; + + private final AtomicBoolean metadataWritten = new AtomicBoolean(false); + private final AtomicBoolean published = new AtomicBoolean(false); + + public DumpWriterImpl(ProjectDescriptor descriptor, ProjectExportDumpFS projectExportDumpFS, TempFolder tempFolder) { + this.descriptor = descriptor; + this.projectExportDumpFS = projectExportDumpFS; + this.tempFolder = tempFolder; + this.rootDir = tempFolder.newDir(); + } + + @Override + public void write(ProjectDump.Metadata metadata) { + checkNotPublished(); + if (metadataWritten.get()) { + throw new IllegalStateException("Metadata has already been written"); + } + File file = new File(rootDir, METADATA.filename()); + try (FileOutputStream output = FILES2.openOutputStream(file, false)) { + PROTOBUF2.writeTo(metadata, output); + metadataWritten.set(true); + } catch (IOException e) { + throw new IllegalStateException("Can not write to file " + file, e); + } + } + + @Override + public StreamWriter newStreamWriter(DumpElement elt) { + checkNotPublished(); + File file = new File(rootDir, elt.filename()); + return StreamWriterImpl.create(file); + } + + @Override + public void publish() { + checkNotPublished(); + if (!metadataWritten.get()) { + throw new IllegalStateException("Metadata is missing"); + } + File zip = tempFolder.newFile(); + FILES2.zipDir(rootDir, zip); + + File targetZip = projectExportDumpFS.exportDumpOf(descriptor); + FILES2.deleteIfExists(targetZip); + FILES2.moveFile(zip, targetZip); + FILES2.deleteIfExists(rootDir); + Loggers.get(getClass()).info("Dump file published | size={} | path={}", humanReadableByteCountSI(sizeOf(targetZip)), targetZip.getAbsolutePath()); + published.set(true); + } + + private void checkNotPublished() { + if (published.get()) { + throw new IllegalStateException("Dump is already published"); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportEventsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportEventsStep.java new file mode 100644 index 00000000000..5883371ad95 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportEventsStep.java @@ -0,0 +1,116 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.SnapshotDto; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; +import static org.sonar.db.DatabaseUtils.getString; + +public class ExportEventsStep implements ComputationStep { + + private static final String QUERY = "select" + + " p.uuid, e.name, e.analysis_uuid, e.category, e.description, e.event_data, e.event_date, e.uuid" + + " from events e" + + " join snapshots s on s.uuid=e.analysis_uuid" + + " join components p on p.uuid=s.component_uuid" + + " join project_branches pb on pb.uuid=p.uuid" + + " where pb.project_uuid=? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge=? and s.status=? and p.enabled=?"; + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final ComponentRepository componentRepository; + private final DumpWriter dumpWriter; + + public ExportEventsStep(DbClient dbClient, ProjectHolder projectHolder, ComponentRepository componentRepository, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.componentRepository = componentRepository; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.EVENTS); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = buildSelectStatement(dbSession); + ResultSet rs = stmt.executeQuery()) { + + ProjectDump.Event.Builder builder = ProjectDump.Event.newBuilder(); + while (rs.next()) { + ProjectDump.Event event = convertToEvent(rs, builder); + output.write(event); + count++; + } + Loggers.get(getClass()).debug("{} events exported", count); + + } catch (Exception e) { + throw new IllegalStateException(format("Event Export failed after processing %d events successfully", count), e); + } + } + + private PreparedStatement buildSelectStatement(DbSession dbSession) throws SQLException { + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY); + try { + stmt.setString(1, projectHolder.projectDto().getUuid()); + stmt.setBoolean(2, true); + stmt.setString(3, SnapshotDto.STATUS_PROCESSED); + stmt.setBoolean(4, true); + return stmt; + } catch (Exception e) { + DatabaseUtils.closeQuietly(stmt); + throw e; + } + } + + private ProjectDump.Event convertToEvent(ResultSet rs, ProjectDump.Event.Builder builder) throws SQLException { + long componentRef = componentRepository.getRef(rs.getString(1)); + return builder + .clear() + .setComponentRef(componentRef) + .setName(defaultString(getString(rs, 2))) + .setAnalysisUuid(getString(rs, 3)) + .setCategory(defaultString(getString(rs, 4))) + .setDescription(defaultString(getString(rs, 5))) + .setData(defaultString(getString(rs, 6))) + .setDate(rs.getLong(7)) + .setUuid(rs.getString(8)) + .build(); + + } + + @Override + public String getDescription() { + return "Export events"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLinksStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLinksStep.java new file mode 100644 index 00000000000..20bfce7b43f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLinksStep.java @@ -0,0 +1,81 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.ProjectLinkDto; +import org.sonar.db.project.ProjectExportMapper; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; + +public class ExportLinksStep implements ComputationStep { + + private final DbClient dbClient; + private final ComponentRepository componentRepository; + private final ProjectHolder projectHolder; + private final DumpWriter dumpWriter; + + public ExportLinksStep(DbClient dbClient, ComponentRepository componentRepository, ProjectHolder projectHolder, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.componentRepository = componentRepository; + this.projectHolder = projectHolder; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + + try { + try (DbSession dbSession = dbClient.openSession(false); + StreamWriter linksWriter = dumpWriter.newStreamWriter(DumpElement.LINKS)) { + ProjectDump.Link.Builder builder = ProjectDump.Link.newBuilder(); + List links = dbSession.getMapper(ProjectExportMapper.class).selectLinksForExport(projectHolder.projectDto().getUuid()); + for (ProjectLinkDto link : links) { + builder + .clear() + .setUuid(link.getUuid()) + .setName(defaultString(link.getName())) + .setHref(defaultString(link.getHref())) + .setType(defaultString(link.getType())) + .setComponentRef(componentRepository.getRef(link.getProjectUuid())); + linksWriter.write(builder.build()); + ++count; + } + + Loggers.get(getClass()).debug("{} links exported", count); + } + } catch (Exception e) { + throw new IllegalStateException(format("Link export failed after processing %d link(s) successfully", count), e); + } + } + + @Override + public String getDescription() { + return "Export links"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStep.java new file mode 100644 index 00000000000..96855d881a4 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStep.java @@ -0,0 +1,112 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; +import static org.sonar.db.DatabaseUtils.getDouble; +import static org.sonar.db.DatabaseUtils.getString; + +public class ExportLiveMeasuresStep implements ComputationStep { + + private static final String QUERY = "select pm.metric_uuid, pm.component_uuid, pm.text_value, pm.value, pm.variation" + + " from live_measures pm" + + " join metrics m on m.uuid=pm.metric_uuid" + + " join components p on p.uuid = pm.component_uuid" + + " join components pp on pp.uuid = pm.project_uuid" + + " join project_branches pb on pb.uuid=pp.uuid" + + " where pb.project_uuid=? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge=? and p.enabled=? and m.enabled=?"; + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final ComponentRepository componentRepository; + private final MutableMetricRepository metricHolder; + private final DumpWriter dumpWriter; + + public ExportLiveMeasuresStep(DbClient dbClient, ProjectHolder projectHolder, ComponentRepository componentRepository, + MutableMetricRepository metricHolder, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.componentRepository = componentRepository; + this.metricHolder = metricHolder; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.LIVE_MEASURES); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY)) { + stmt.setString(1, projectHolder.projectDto().getUuid()); + stmt.setBoolean(2, true); + stmt.setBoolean(3, true); + stmt.setBoolean(4, true); + try (ResultSet rs = stmt.executeQuery()) { + ProjectDump.LiveMeasure.Builder liveMeasureBuilder = ProjectDump.LiveMeasure.newBuilder(); + ProjectDump.DoubleValue.Builder doubleBuilder = ProjectDump.DoubleValue.newBuilder(); + while (rs.next()) { + ProjectDump.LiveMeasure measure = convertToLiveMeasure(rs, liveMeasureBuilder, doubleBuilder); + output.write(measure); + count++; + } + Loggers.get(getClass()).debug("{} live measures exported", count); + } + } catch (Exception e) { + throw new IllegalStateException(format("Live Measure Export failed after processing %d measures successfully", count), e); + } + } + + private ProjectDump.LiveMeasure convertToLiveMeasure(ResultSet rs, ProjectDump.LiveMeasure.Builder builder, + ProjectDump.DoubleValue.Builder doubleBuilder) throws SQLException { + long componentRef = componentRepository.getRef(rs.getString(2)); + int metricRef = metricHolder.add(rs.getString(1)); + builder + .clear() + .setMetricRef(metricRef) + .setComponentRef(componentRef) + .setTextValue(defaultString(getString(rs, 3))); + Double value = getDouble(rs, 4); + if (value != null) { + builder.setDoubleValue(doubleBuilder.setValue(value).build()); + } + Double variation = getDouble(rs, 5); + if (variation != null) { + builder.setVariation(doubleBuilder.setValue(variation).build()); + } + return builder.build(); + } + + @Override + public String getDescription() { + return "Export live measures"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStep.java new file mode 100644 index 00000000000..615c3c1b096 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStep.java @@ -0,0 +1,131 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.SnapshotDto; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; +import static org.sonar.db.DatabaseUtils.getDouble; +import static org.sonar.db.DatabaseUtils.getString; + +public class ExportMeasuresStep implements ComputationStep { + + private static final String QUERY = "select pm.metric_uuid, pm.analysis_uuid, pm.component_uuid, pm.text_value, pm.value," + + " pm.alert_status, pm.alert_text, pm.variation_value_1" + + " from project_measures pm" + + " join metrics m on m.uuid=pm.metric_uuid" + + " join snapshots s on s.uuid=pm.analysis_uuid" + + " join components p on p.uuid=pm.component_uuid" + + " join project_branches pb on pb.uuid=p.uuid" + + " where pb.project_uuid=? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge=?" + + " and s.status=? and p.enabled=? and m.enabled=? and pm.person_id is null"; + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final ComponentRepository componentRepository; + private final MutableMetricRepository metricHolder; + private final DumpWriter dumpWriter; + + public ExportMeasuresStep(DbClient dbClient, ProjectHolder projectHolder, ComponentRepository componentRepository, MutableMetricRepository metricHolder, + DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.componentRepository = componentRepository; + this.metricHolder = metricHolder; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.MEASURES); + DbSession dbSession = dbClient.openSession(false); + PreparedStatement stmt = createSelectStatement(dbSession); + ResultSet rs = stmt.executeQuery()) { + + ProjectDump.Measure.Builder measureBuilder = ProjectDump.Measure.newBuilder(); + ProjectDump.DoubleValue.Builder doubleBuilder = ProjectDump.DoubleValue.newBuilder(); + while (rs.next()) { + ProjectDump.Measure measure = convertToMeasure(rs, measureBuilder, doubleBuilder); + output.write(measure); + count++; + } + Loggers.get(getClass()).debug("{} measures exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Measure Export failed after processing %d measures successfully", count), e); + } + } + + private PreparedStatement createSelectStatement(DbSession dbSession) throws SQLException { + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY); + try { + stmt.setString(1, projectHolder.projectDto().getUuid()); + stmt.setBoolean(2, true); + stmt.setString(3, SnapshotDto.STATUS_PROCESSED); + stmt.setBoolean(4, true); + stmt.setBoolean(5, true); + return stmt; + } catch (Exception t) { + DatabaseUtils.closeQuietly(stmt); + throw t; + } + } + + private ProjectDump.Measure convertToMeasure(ResultSet rs, ProjectDump.Measure.Builder builder, + ProjectDump.DoubleValue.Builder doubleBuilder) throws SQLException { + long componentRef = componentRepository.getRef(rs.getString(3)); + int metricRef = metricHolder.add(rs.getString(1)); + + builder + .clear() + .setMetricRef(metricRef) + .setAnalysisUuid(rs.getString(2)) + .setComponentRef(componentRef) + .setTextValue(defaultString(getString(rs, 4))); + Double value = getDouble(rs, 5); + if (value != null) { + builder.setDoubleValue(doubleBuilder.setValue(value).build()); + } + builder.setAlertStatus(defaultString(getString(rs, 6))); + builder.setAlertText(defaultString(getString(rs, 7))); + Double var1 = getDouble(rs, 8); + if (var1 != null) { + builder.setVariation1(doubleBuilder.setValue(var1).build()); + } + return builder.build(); + } + + @Override + public String getDescription() { + return "Export measures"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStep.java new file mode 100644 index 00000000000..b0b925f082a --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStep.java @@ -0,0 +1,75 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import java.util.Map; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.metric.MetricDto; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; + +public class ExportMetricsStep implements ComputationStep { + + private final DbClient dbClient; + private final MetricRepository metricsHolder; + private final DumpWriter dumpWriter; + + public ExportMetricsStep(DbClient dbClient, MetricRepository metricsHolder, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.metricsHolder = metricsHolder; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + int count = 0; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.METRICS); + DbSession dbSession = dbClient.openSession(false)) { + + ProjectDump.Metric.Builder builder = ProjectDump.Metric.newBuilder(); + Map refByUuid = metricsHolder.getRefByUuid(); + List dtos = dbClient.metricDao().selectByUuids(dbSession, refByUuid.keySet()); + for (MetricDto dto : dtos) { + builder + .clear() + .setRef(refByUuid.get(dto.getUuid())) + .setKey(dto.getKey()) + .setName(defaultString(dto.getShortName())); + output.write(builder.build()); + count++; + } + Loggers.get(getClass()).debug("{} metrics exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Metric Export failed after processing %d metrics successfully", count), e); + } + } + + @Override + public String getDescription() { + return "Export metrics"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStep.java new file mode 100644 index 00000000000..038c511a2b5 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStep.java @@ -0,0 +1,82 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.newcodeperiod.NewCodePeriodDto; +import org.sonar.db.project.ProjectExportMapper; + +import static java.lang.String.format; + +public class ExportNewCodePeriodsStep implements ComputationStep { + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final DumpWriter dumpWriter; + + public ExportNewCodePeriodsStep(DbClient dbClient, ProjectHolder projectHolder, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.NEW_CODE_PERIODS); + DbSession dbSession = dbClient.openSession(false)) { + + final ProjectDump.NewCodePeriod.Builder builder = ProjectDump.NewCodePeriod.newBuilder(); + final List newCodePeriods = dbSession.getMapper(ProjectExportMapper.class) + .selectNewCodePeriodsForExport(projectHolder.projectDto().getUuid()); + for (NewCodePeriodDto newCodePeriod : newCodePeriods) { + builder.clear() + .setUuid(newCodePeriod.getUuid()) + .setProjectUuid(newCodePeriod.getProjectUuid()) + .setType(newCodePeriod.getType().name()); + + if (newCodePeriod.getBranchUuid() != null) { + builder.setBranchUuid(newCodePeriod.getBranchUuid()); + } + + if (newCodePeriod.getValue() != null) { + builder.setValue(newCodePeriod.getValue()); + } + output.write(builder.build()); + ++count; + } + + Loggers.get(getClass()).debug("{} new code periods exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("New Code Periods Export failed after processing %d new code periods successfully", count), e); + } + } + + @Override + public String getDescription() { + return "Export new code periods"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStep.java new file mode 100644 index 00000000000..03da3f7ba78 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStep.java @@ -0,0 +1,70 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.Collection; +import org.apache.commons.lang.StringUtils; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.core.platform.PluginInfo; +import org.sonar.core.platform.PluginRepository; +import org.sonar.updatecenter.common.Version; + +public class ExportPluginsStep implements ComputationStep { + + private final PluginRepository pluginRepository; + private final DumpWriter dumpWriter; + + public ExportPluginsStep(PluginRepository pluginRepository, DumpWriter dumpWriter) { + this.pluginRepository = pluginRepository; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + try (StreamWriter writer = dumpWriter.newStreamWriter(DumpElement.PLUGINS)) { + + Collection plugins = pluginRepository.getPluginInfos(); + for (PluginInfo plugin : plugins) { + ProjectDump.Plugin.Builder builder = ProjectDump.Plugin.newBuilder(); + writer.write(convert(plugin, builder)); + } + Loggers.get(getClass()).debug("{} plugins exported", plugins.size()); + } + } + + private static ProjectDump.Plugin convert(PluginInfo plugin, ProjectDump.Plugin.Builder builder) { + builder + .clear() + .setKey(plugin.getKey()) + .setName(StringUtils.defaultString(plugin.getName())); + Version version = plugin.getVersion(); + if (version != null) { + builder.setVersion(version.toString()); + } + return builder.build(); + } + + @Override + public String getDescription() { + return "Export plugins"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStep.java new file mode 100644 index 00000000000..70b74b15f5e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStep.java @@ -0,0 +1,93 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.common.collect.ImmutableSet; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.project.ProjectExportMapper; +import org.sonar.db.property.PropertyDto; + +import static java.lang.String.format; +import static org.apache.commons.lang.StringUtils.defaultString; + +public class ExportSettingsStep implements ComputationStep { + + /** + * These properties are not exported as values depend on environment data that are not + * exported within the dump (Quality Gate, users). + */ + private static final Set IGNORED_KEYS = ImmutableSet.of("sonar.issues.defaultAssigneeLogin"); + + private final DbClient dbClient; + private final ProjectHolder projectHolder; + private final ComponentRepository componentRepository; + private final DumpWriter dumpWriter; + + public ExportSettingsStep(DbClient dbClient, ProjectHolder projectHolder, ComponentRepository componentRepository, DumpWriter dumpWriter) { + this.dbClient = dbClient; + this.projectHolder = projectHolder; + this.componentRepository = componentRepository; + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + long count = 0L; + try ( + StreamWriter output = dumpWriter.newStreamWriter(DumpElement.SETTINGS); + DbSession dbSession = dbClient.openSession(false)) { + + final ProjectDump.Setting.Builder builder = ProjectDump.Setting.newBuilder(); + final List properties = dbSession.getMapper(ProjectExportMapper.class).selectPropertiesForExport(projectHolder.projectDto().getUuid()) + .stream() + .filter(dto -> dto.getComponentUuid() != null) + .filter(dto -> !IGNORED_KEYS.contains(dto.getKey())) + .collect(Collectors.toList()); + for (PropertyDto property : properties) { + builder.clear() + .setKey(property.getKey()) + .setValue(defaultString(property.getValue())); + + if (property.getComponentUuid() != null) { + builder.setComponentRef(componentRepository.getRef(property.getComponentUuid())); + } + output.write(builder.build()); + ++count; + } + + Loggers.get(getClass()).debug("{} settings exported", count); + } catch (Exception e) { + throw new IllegalStateException(format("Settings Export failed after processing %d settings successfully", count), e); + } + } + + @Override + public String getDescription() { + return "Export settings"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/LoadProjectStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/LoadProjectStep.java new file mode 100644 index 00000000000..cdef363c63e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/LoadProjectStep.java @@ -0,0 +1,66 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import java.util.List; +import org.sonar.api.utils.MessageException; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.BranchDto; +import org.sonar.db.project.ProjectDto; + +import static java.lang.String.format; +import static org.sonar.core.util.stream.MoreCollectors.toList; + +/** + * Loads project from database and verifies that it's valid: it must exist and be a project ! + */ +public class LoadProjectStep implements ComputationStep { + + private final ProjectDescriptor descriptor; + private final MutableProjectHolder definitionHolder; + private final DbClient dbClient; + + public LoadProjectStep(ProjectDescriptor descriptor, MutableProjectHolder definitionHolder, + DbClient dbClient) { + this.descriptor = descriptor; + this.definitionHolder = definitionHolder; + this.dbClient = dbClient; + } + + @Override + public void execute(Context context) { + try (DbSession dbSession = dbClient.openSession(false)) { + ProjectDto project = dbClient.projectDao().selectProjectByKey(dbSession, descriptor.getKey()) + .orElseThrow(() -> MessageException.of(format("Project with key [%s] does not exist", descriptor.getKey()))); + definitionHolder.setProjectDto(project); + + List branches = dbClient.branchDao().selectByProject(dbSession, project).stream().collect(toList()); + definitionHolder.setBranches(branches); + } + } + + @Override + public String getDescription() { + return "Load project"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStream.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStream.java new file mode 100644 index 00000000000..b9e18f472b7 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStream.java @@ -0,0 +1,29 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; + +public interface MessageStream extends Iterable, AutoCloseable { + + @Override + void close(); + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStreamImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStreamImpl.java new file mode 100644 index 00000000000..11b47979a59 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MessageStreamImpl.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; +import com.google.protobuf.Parser; +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.util.Iterator; +import javax.annotation.CheckForNull; +import org.sonar.api.utils.System2; +import org.sonar.core.util.CloseableIterator; + +import static org.sonar.ce.task.util.Protobuf2.PROTOBUF2; + +public class MessageStreamImpl implements MessageStream { + + private final InputStream input; + private final Parser parser; + + public MessageStreamImpl(InputStream input, Parser parser) { + this.input = new BufferedInputStream(input); + this.parser = parser; + } + + @Override + public Iterator iterator() { + return new StreamMessagesIterator(); + } + + @Override + public void close() { + System2.INSTANCE.close(input); + } + + private class StreamMessagesIterator extends CloseableIterator { + @CheckForNull + @Override + protected M doNext() { + return PROTOBUF2.parseDelimitedFrom(parser, input); + } + + @Override + protected void doClose() { + // no need, already handled by method close() of outer class + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MetricRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MetricRepository.java new file mode 100644 index 00000000000..f9cab6439e4 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MetricRepository.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import java.util.Map; + +/** + * Database uuids of the metrics related to exported measures. + */ +public interface MetricRepository { + + /** + * @return map of db uuids + */ + Map getRefByUuid(); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReader.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReader.java new file mode 100644 index 00000000000..0dd29aa5886 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReader.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import java.io.File; + +public interface MutableDumpReader extends DumpReader { + void setZipFile(File file); + + void setTempRootDir(File d); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReaderImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReaderImpl.java new file mode 100644 index 00000000000..e6ef3605f42 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableDumpReaderImpl.java @@ -0,0 +1,88 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static org.sonar.ce.task.projectexport.steps.DumpElement.METADATA; +import static org.sonar.ce.task.util.Files2.FILES2; +import static org.sonar.ce.task.util.Protobuf2.PROTOBUF2; + +public class MutableDumpReaderImpl implements MutableDumpReader { + + private File zipFile = null; + private File tempRootDir = null; + + @Override + public void setZipFile(File file) { + checkNotNull(file, "Project dump file can not be null"); + checkArgument(file.exists(), "Project dump file does not exist: %s", file); + checkArgument(file.isFile(), "Project dump is not a file: %s", file); + this.zipFile = file; + } + + @Override + public void setTempRootDir(File d) { + checkNotNull(d, "Dump extraction directory can not be null"); + checkArgument(d.exists(), "Dump extraction directory does not exist: %s", d); + checkArgument(d.isDirectory(), "Dump extraction is not a directory: %s", d); + this.tempRootDir = d; + } + + @Override + public File zipFile() { + checkState(zipFile != null, "Project dump file has not been set"); + return zipFile; + } + + @Override + public File tempRootDir() { + checkState(tempRootDir != null, "Dump file has not been extracted"); + return tempRootDir; + } + + @Override + public ProjectDump.Metadata metadata() { + File file = new File(tempRootDir(), METADATA.filename()); + checkState(file.exists(), "Missing metadata file: %s", file); + try (FileInputStream input = FILES2.openInputStream(file)) { + return PROTOBUF2.parseFrom(METADATA.parser(), input); + } catch (IOException e) { + throw new IllegalStateException("Can not read file " + file, e); + } + } + + @Override + public MessageStream stream(DumpElement elt) { + File file = new File(tempRootDir(), elt.filename()); + checkState(file.exists(), "Missing file: %s", file); + + InputStream input = FILES2.openInputStream(file); + return new MessageStreamImpl<>(input, elt.parser()); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepository.java new file mode 100644 index 00000000000..937b62da2d3 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepository.java @@ -0,0 +1,26 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +public interface MutableMetricRepository extends MetricRepository { + + int add(String metricUuid); + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImpl.java new file mode 100644 index 00000000000..dc507a87e06 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImpl.java @@ -0,0 +1,39 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import java.util.HashMap; +import java.util.Map; + +public class MutableMetricRepositoryImpl implements MutableMetricRepository { + + private final Map refByUuid = new HashMap<>(); + private int ref = 0; + + @Override + public Map getRefByUuid() { + return refByUuid; + } + + @Override + public int add(String metricUuid) { + return refByUuid.computeIfAbsent(metricUuid, uuid -> ref++); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolder.java new file mode 100644 index 00000000000..64d70be6e5c --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolder.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import java.util.List; +import org.sonar.db.component.BranchDto; +import org.sonar.db.project.ProjectDto; + +public interface MutableProjectHolder extends ProjectHolder { + + void setProjectDto(ProjectDto dto); + + void setBranches(List branches); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolderImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolderImpl.java new file mode 100644 index 00000000000..42d342cc999 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/MutableProjectHolderImpl.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import java.util.Collections; +import java.util.List; +import org.sonar.db.component.BranchDto; +import org.sonar.db.project.ProjectDto; + +import static com.google.common.collect.ImmutableList.copyOf; +import static java.util.Objects.requireNonNull; + +public class MutableProjectHolderImpl implements MutableProjectHolder { + + private ProjectDto projectDto = null; + + private List branches = Collections.emptyList(); + + @Override + public void setProjectDto(ProjectDto dto) { + requireNonNull(dto, "Project must not be null"); + this.projectDto = dto; + } + + @Override + public void setBranches(List branches) { + requireNonNull(branches, "Branches must not be null"); + this.branches = copyOf(branches); + } + + @Override + public ProjectDto projectDto() { + requireNonNull(projectDto, "Project has not been loaded yet"); + return projectDto; + } + + @Override + public List branches() { + return branches; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ProjectHolder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ProjectHolder.java new file mode 100644 index 00000000000..0fa7c0ebf00 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/ProjectHolder.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import java.util.List; +import org.sonar.db.component.BranchDto; +import org.sonar.db.project.ProjectDto; + +public interface ProjectHolder { + + ProjectDto projectDto(); + + List branches(); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/PublishDumpStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/PublishDumpStep.java new file mode 100644 index 00000000000..8ab6f0dbec5 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/PublishDumpStep.java @@ -0,0 +1,44 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import org.sonar.ce.task.step.ComputationStep; + +/** + * Zips the dump file and makes it available to users. + */ +public class PublishDumpStep implements ComputationStep { + + private final DumpWriter dumpWriter; + + public PublishDumpStep(DumpWriter dumpWriter) { + this.dumpWriter = dumpWriter; + } + + @Override + public void execute(Context context) { + dumpWriter.publish(); + } + + @Override + public String getDescription() { + return "Publish dump file"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriter.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriter.java new file mode 100644 index 00000000000..11893e74faf --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriter.java @@ -0,0 +1,29 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; + +public interface StreamWriter extends AutoCloseable { + void write(M msg); + + @Override + void close(); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriterImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriterImpl.java new file mode 100644 index 00000000000..1a63841fa24 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/StreamWriterImpl.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.protobuf.Message; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import org.sonar.api.utils.System2; + +import static org.sonar.ce.task.util.Files2.FILES2; +import static org.sonar.ce.task.util.Protobuf2.PROTOBUF2; + +public class StreamWriterImpl implements StreamWriter { + + private final OutputStream output; + + private StreamWriterImpl(OutputStream output) { + this.output = new BufferedOutputStream(output); + } + + @Override + public void write(M message) { + PROTOBUF2.writeDelimitedTo(message, output); + } + + @Override + public void close() { + System2.INSTANCE.close(output); + } + + public static StreamWriterImpl create(File file) { + FileOutputStream output = FILES2.openOutputStream(file, true); + return new StreamWriterImpl<>(output); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStep.java new file mode 100644 index 00000000000..309a537c933 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStep.java @@ -0,0 +1,55 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import org.sonar.api.SonarRuntime; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.step.ComputationStep; + +public class WriteMetadataStep implements ComputationStep { + + private final System2 system2; + private final DumpWriter dumpWriter; + private final ProjectHolder projectHolder; + private final SonarRuntime sonarRuntime; + + public WriteMetadataStep(System2 system2, DumpWriter dumpWriter, ProjectHolder projectHolder, SonarRuntime sonarRuntime) { + this.system2 = system2; + this.dumpWriter = dumpWriter; + this.projectHolder = projectHolder; + this.sonarRuntime = sonarRuntime; + } + + @Override + public void execute(Context context) { + dumpWriter.write(ProjectDump.Metadata.newBuilder() + .setProjectKey(projectHolder.projectDto().getKey()) + .setProjectUuid(projectHolder.projectDto().getUuid()) + .setSonarqubeVersion(sonarRuntime.getApiVersion().toString()) + .setDumpDate(system2.now()) + .build()); + } + + @Override + public String getDescription() { + return "Write metadata"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/package-info.java new file mode 100644 index 00000000000..95a32c6eb44 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/steps/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.steps; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectDescriptor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectDescriptor.java new file mode 100644 index 00000000000..63e12cb87c2 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectDescriptor.java @@ -0,0 +1,85 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.taskprocessor; + +import java.util.Objects; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import org.sonar.db.component.ComponentDto; + +import static java.util.Objects.requireNonNull; + +@Immutable +public class ProjectDescriptor { + private final String uuid; + private final String key; + private final String name; + + public ProjectDescriptor(String uuid, String key, String name) { + this.uuid = requireNonNull(uuid); + this.key = requireNonNull(key); + this.name = requireNonNull(name); + } + + /** + * Build a {@link ProjectDescriptor} without checking qualifier of ComponentDto. + */ + public static ProjectDescriptor of(ComponentDto project) { + return new ProjectDescriptor(project.uuid(), project.getDbKey(), project.name()); + } + + public final String getUuid() { + return uuid; + } + + public final String getKey() { + return key; + } + + public final String getName() { + return name; + } + + @Override + public final boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProjectDescriptor that = (ProjectDescriptor) o; + return key.equals(that.key); + } + + @Override + public final int hashCode() { + return Objects.hash(key); + } + + @Override + public final String toString() { + return getClass().getSimpleName() + "{" + + "uuid='" + uuid + '\'' + + ", key='" + key + '\'' + + ", name='" + name + '\'' + + '}'; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectExportTaskProcessor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectExportTaskProcessor.java new file mode 100644 index 00000000000..0cb894029fc --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/taskprocessor/ProjectExportTaskProcessor.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.taskprocessor; + +import java.util.Set; +import org.sonar.ce.task.CeTask; +import org.sonar.ce.task.CeTaskResult; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.container.TaskContainerImpl; +import org.sonar.ce.task.projectexport.ProjectExportContainerPopulator; +import org.sonar.ce.task.projectexport.ProjectExportProcessor; +import org.sonar.ce.task.taskprocessor.CeTaskProcessor; +import org.sonar.core.platform.ComponentContainer; + +import static org.sonar.db.ce.CeTaskTypes.PROJECT_EXPORT; + +public class ProjectExportTaskProcessor implements CeTaskProcessor { + + private final ComponentContainer componentContainer; + + public ProjectExportTaskProcessor(ComponentContainer componentContainer) { + this.componentContainer = componentContainer; + } + + @Override + public Set getHandledCeTaskTypes() { + return Set.of(PROJECT_EXPORT); + } + + @Override + public CeTaskResult process(CeTask task) { + processProjectExport(task); + return null; + } + + private void processProjectExport(CeTask task) { + CeTask.Component exportComponent = mandatoryComponent(task, PROJECT_EXPORT); + failIfNotMain(exportComponent, task); + ProjectDescriptor projectExportDescriptor = new ProjectDescriptor(exportComponent.getUuid(), + mandatoryKey(exportComponent), mandatoryName(exportComponent)); + + try (TaskContainer taskContainer = new TaskContainerImpl(componentContainer, + new ProjectExportContainerPopulator(projectExportDescriptor))) { + taskContainer.bootup(); + taskContainer.getComponentByType(ProjectExportProcessor.class).process(); + } + + } + + private static void failIfNotMain(CeTask.Component exportComponent, CeTask task) { + task.getMainComponent().filter(mainComponent -> mainComponent.equals(exportComponent)) + .orElseThrow(() -> new IllegalStateException("Component of task must be the same as main component")); + } + + private static CeTask.Component mandatoryComponent(CeTask task, String type) { + return task.getComponent().orElseThrow(() -> new IllegalStateException(String.format("Task with type %s must have a component", type))); + } + + private static String mandatoryKey(CeTask.Component component) { + return component.getKey().orElseThrow(() -> new IllegalStateException("Task component must have a key")); + } + + private static String mandatoryName(CeTask.Component component) { + return component.getName().orElseThrow(() -> new IllegalStateException("Task component must have a name")); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFS.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFS.java new file mode 100644 index 00000000000..56f344928ce --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFS.java @@ -0,0 +1,34 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.util; + +import java.io.File; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; + +public interface ProjectExportDumpFS { + + /** + * Create a File representing the exported dump of the specified {@link ProjectDescriptor}. + *

+ * This method doesn't check nor enforce the File actually exist and/or is a regular file. + *

+ */ + File exportDumpOf(ProjectDescriptor descriptor); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImpl.java new file mode 100644 index 00000000000..e224587816d --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImpl.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.util; + +import java.io.File; +import org.picocontainer.Startable; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.config.Configuration; +import org.sonar.api.server.ServerSide; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.util.Files2; + +import static org.sonar.core.util.Slug.slugify; +import static org.sonar.process.ProcessProperties.Property.PATH_DATA; + +@ServerSide +@ComputeEngineSide +public class ProjectExportDumpFSImpl implements ProjectExportDumpFS, Startable { + private static final String GOVERNANCE_DIR_NAME = "governance"; + private static final String PROJECT_DUMPS_DIR_NAME = "project_dumps"; + private static final String DUMP_FILE_EXTENSION = ".zip"; + + private final File exportDir; + + public ProjectExportDumpFSImpl(Configuration config) { + String dataPath = config.get(PATH_DATA.getKey()).get(); + File governanceDir = new File(dataPath, GOVERNANCE_DIR_NAME); + File projectDumpDir = new File(governanceDir, PROJECT_DUMPS_DIR_NAME); + this.exportDir = new File(projectDumpDir, "export"); + } + + @Override + public void start() { + Files2.FILES2.createDir(importDir); + Files2.FILES2.createDir(exportDir); + } + + @Override + public void stop() { + // nothing to do + } + + @Override + public File exportDumpOf(ProjectDescriptor descriptor) { + String fileName = slugify(descriptor.getKey()) + DUMP_FILE_EXTENSION; + return new File(exportDir, fileName); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFS.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFS.java new file mode 100644 index 00000000000..21202300216 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFS.java @@ -0,0 +1,34 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.util; + +import java.io.File; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; + +public interface ProjectImportDumpFS { + /** + * Create a File representing the archive dump of the specified {@link ProjectDescriptor} to be imported. + *

+ * This method doesn't check nor enforce the File actually exist and/or is a regular file. + *

+ */ + File importDumpOf(ProjectDescriptor descriptor); + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImpl.java new file mode 100644 index 00000000000..fadb16f98e9 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImpl.java @@ -0,0 +1,66 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.util; + +import java.io.File; +import org.picocontainer.Startable; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.config.Configuration; +import org.sonar.api.server.ServerSide; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.util.Files2; + +import static org.sonar.core.util.Slug.slugify; +import static org.sonar.process.ProcessProperties.Property.PATH_DATA; + +@ServerSide +@ComputeEngineSide +public class ProjectImportDumpFSImpl implements ProjectImportDumpFS, Startable { + private static final String GOVERNANCE_DIR_NAME = "governance"; + private static final String PROJECT_DUMPS_DIR_NAME = "project_dumps"; + private static final String DUMP_FILE_EXTENSION = ".zip"; + + private final File importDir; + + public ProjectImportDumpFSImpl(Configuration config) { + String dataPath = config.get(PATH_DATA.getKey()).get(); + File governanceDir = new File(dataPath, GOVERNANCE_DIR_NAME); + File projectDumpDir = new File(governanceDir, PROJECT_DUMPS_DIR_NAME); + this.importDir = new File(projectDumpDir, "import"); + } + + @Override + public void start() { + Files2.FILES2.createDir(importDir); + Files2.FILES2.createDir(exportDir); + } + + @Override + public void stop() { + // nothing to do + } + + @Override + public File importDumpOf(ProjectDescriptor descriptor) { + String fileName = slugify(descriptor.getKey()) + DUMP_FILE_EXTENSION; + return new File(importDir, fileName); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ResultSetUtils.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ResultSetUtils.java new file mode 100644 index 00000000000..5adbc08cf0b --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/ResultSetUtils.java @@ -0,0 +1,61 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.util; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public final class ResultSetUtils { + private ResultSetUtils() { + // prevents instantiation + } + + public static int defaultIfNull(ResultSet rs, int columnIndex, int defaultValue) throws SQLException { + int value = rs.getInt(columnIndex); + if (rs.wasNull()) { + return defaultValue; + } + return value; + } + + public static double defaultIfNull(ResultSet rs, int columnIndex, double defaultValue) throws SQLException { + double value = rs.getDouble(columnIndex); + if (rs.wasNull()) { + return defaultValue; + } + return value; + } + + public static long defaultIfNull(ResultSet rs, int columnIndex, long defaultValue) throws SQLException { + long value = rs.getLong(columnIndex); + if (rs.wasNull()) { + return defaultValue; + } + return value; + } + + public static String emptyIfNull(ResultSet rs, int columnIndex) throws SQLException { + String value = rs.getString(columnIndex); + if (value == null) { + return ""; + } + return value; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/package-info.java new file mode 100644 index 00000000000..7f306f4d00f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/util/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectexport.util; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Files2.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Files2.java new file mode 100644 index 00000000000..740c7b34335 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Files2.java @@ -0,0 +1,298 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.annotation.Nullable; +import org.sonar.api.utils.ZipUtils; +import org.sonar.core.util.FileUtils; + +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import static java.nio.file.Files.createDirectories; +import static java.util.Objects.requireNonNull; + +/** + * This utility class provides Java NIO based replacement for some methods of + * {@link org.apache.commons.io.FileUtils Common IO FileUtils} class. + * Only the methods which name ends with {@code "orThrowIOE"} may raise {@link IOException}. + * Others wrap checked exceptions into {@link IllegalStateException}. + */ +public class Files2 { + + public static final Files2 FILES2 = new Files2(); + + private Files2() { + // use FILES2 singleton + } + + /** + * Deletes a directory or a file if it exists, else does nothing. In the case + * of a directly, it is deleted recursively. + * + * @param fileOrDir file or directory to delete + * @throws IOException in case deletion is unsuccessful + */ + public void deleteIfExistsOrThrowIOE(File fileOrDir) throws IOException { + if (!fileOrDir.exists()) { + return; + } + if (fileOrDir.isDirectory()) { + FileUtils.deleteDirectory(fileOrDir); + } else { + Files.delete(fileOrDir.toPath()); + } + } + + /** + * Like {@link #deleteIfExistsOrThrowIOE(File)} but wraps {@link IOException} + * into {@link IllegalStateException}. + * + * @throws IllegalStateException in case deletion is unsuccessful + */ + public void deleteIfExists(File fileOrDir) { + try { + deleteIfExistsOrThrowIOE(fileOrDir); + } catch (IOException e) { + throw new IllegalStateException("Can not delete " + fileOrDir, e); + } + } + + /** + * Deletes a directory or a file if it exists, else does nothing. In the case + * of a directly, it is deleted recursively. Any exception is trapped and + * ignored. + * + * @param fileOrDir file or directory to delete. Can be {@code null}. + */ + public void deleteQuietly(@Nullable File fileOrDir) { + FileUtils.deleteQuietly(fileOrDir); + } + + /** + * Moves a file. + * + *

+ * When the destination file is on another file system, do a "copy and delete". + *

+ * + * @param from the file to be moved + * @param to the destination file + * @throws NullPointerException if source or destination is {@code null} + * @throws IOException if the destination file exists + * @throws IOException if source or destination is invalid + * @throws IOException if an IO error occurs moving the file + */ + public void moveFileOrThrowIOE(File from, File to) throws IOException { + org.apache.commons.io.FileUtils.moveFile(from, to); + } + + /** + * Like {@link #moveFileOrThrowIOE(File, File)} but wraps {@link IOException} + * into {@link IllegalStateException}. + * + * @param from the file to be moved + * @param to the destination file + * @throws IllegalStateException if the destination file exists + * @throws IllegalStateException if source or destination is invalid + * @throws IllegalStateException if an IO error occurs moving the file + */ + public void moveFile(File from, File to) { + try { + moveFileOrThrowIOE(from, to); + } catch (IOException e) { + throw new IllegalStateException("Can not move file " + from + " to " + to, e); + } + } + + /** + * Opens a {@link FileOutputStream} for the specified file, checking and + * creating the parent directory if it does not exist. + *

+ * The parent directory will be created if it does not exist. + * The file will be created if it does not exist. + * + * @param file the file to open for output, must not be {@code null} + * @param append if {@code true}, then bytes will be added to the + * end of the file rather than overwriting + * @return a new {@link FileOutputStream} for the specified file + * @throws IOException if the specified file is a directory + * @throws IOException if the file can not be written to + * @throws IOException if a parent directory can not be created + */ + public FileOutputStream openOutputStreamOrThrowIOE(File file, boolean append) throws IOException { + if (file.exists()) { + checkOrThrowIOE(!file.isDirectory(), "File %s exists but is a directory", file); + checkOrThrowIOE(file.canWrite(), "File %s can not be written to", file); + } else { + File parent = file.getParentFile(); + if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { + throw new IOException("Directory " + parent + " could not be created"); + } + } + return new FileOutputStream(file, append); + } + + /** + * Opens a {@link FileOutputStream} for the specified file, checking and + * creating the parent directory if it does not exist. + *

+ * The parent directory will be created if it does not exist. + * The file will be created if it does not exist. + * + * @param file the file to open for output, must not be {@code null} + * @param append if {@code true}, then bytes will be added to the + * end of the file rather than overwriting + * @return a new {@link FileOutputStream} for the specified file + * @throws IllegalStateException if the specified file is a directory + * @throws IllegalStateException if the file can not be written to + * @throws IllegalStateException if a parent directory can not be created + */ + public FileOutputStream openOutputStream(File file, boolean append) { + try { + return openOutputStreamOrThrowIOE(file, append); + } catch (IOException e) { + throw new IllegalStateException("Can not open file " + file, e); + } + } + + /** + * Opens a {@link FileInputStream} for the specified file, providing better + * error messages than simply calling {@code new FileInputStream(file)}. + * + * @param file the file to open, must not be {@code null} + * @return a new {@link FileInputStream} for the specified file + * @throws IOException if the file does not exist + * @throws IOException if the specified file is a directory + * @throws IOException if the file can not be read + */ + public FileInputStream openInputStreamOrThrowIOE(File file) throws IOException { + checkOrThrowIOE(!file.isDirectory(), "File %s exists but is a directory", file); + checkOrThrowIOE(file.exists(), "File %s does not exist", file); + checkOrThrowIOE(file.canRead(), "File %s can not be read", file); + return new FileInputStream(file); + } + + /** + * Opens a {@link FileInputStream} for the specified file, providing better + * error messages than simply calling {@code new FileInputStream(file)}. + * + * @param file the file to open, must not be {@code null} + * @return a new {@link FileInputStream} for the specified file + * @throws IllegalStateException if the file does not exist + * @throws IllegalStateException if the specified file is a directory + * @throws IllegalStateException if the file can not be read + */ + public FileInputStream openInputStream(File file) { + try { + return openInputStreamOrThrowIOE(file); + } catch (IOException e) { + throw new IllegalStateException("Can not open file " + file, e); + } + } + + /** + * Unzips a file to the specified directory. The directory is created if it does not exist. + * + * @throws IOException if {@code zipFile} is a directory + * @throws IOException if {@code zipFile} does not exist + * @throws IOException if {@code toDir} can not be created + */ + public void unzipToDirOrThrowIOE(File zipFile, File toDir) throws IOException { + checkOrThrowIOE(!zipFile.isDirectory(), "File %s exists but is a directory", zipFile); + checkOrThrowIOE(zipFile.exists(), "File %s does not exist", zipFile); + ZipUtils.unzip(zipFile, toDir); + } + + /** + * Unzips a file to the specified directory. The directory is created if it does not exist. + * + * @throws IllegalStateException if {@code zipFile} is a directory + * @throws IllegalStateException if {@code zipFile} does not exist + * @throws IllegalStateException if {@code toDir} can not be created + */ + public void unzipToDir(File zipFile, File toDir) { + try { + unzipToDirOrThrowIOE(zipFile, toDir); + } catch (IOException e) { + throw new IllegalStateException("Can not unzip file " + zipFile + " to directory " + toDir, e); + } + } + + /** + * Zips the directory {@code dir} to the file {@code toFile}. If {@code toFile} is overridden + * if it exists, else it is created. + * + * @throws IllegalStateException if {@code dir} is a not directory + * @throws IllegalStateException if {@code dir} does not exist + * @throws IllegalStateException if {@code toFile} can not be created + */ + public void zipDirOrThrowIOE(File dir, File toFile) throws IOException { + checkOrThrowIOE(dir.exists(), "Directory %s does not exist", dir); + checkOrThrowIOE(dir.isDirectory(), "File %s exists but is not a directory", dir); + ZipUtils.zipDir(dir, toFile); + } + + /** + * Zips the directory {@code dir} to the file {@code toFile}. If {@code toFile} is overridden + * if it exists, else it is created. + * + * @throws IllegalStateException if {@code dir} is a not directory + * @throws IllegalStateException if {@code dir} does not exist + * @throws IllegalStateException if {@code toFile} can not be created + */ + public void zipDir(File dir, File toFile) { + try { + zipDirOrThrowIOE(dir, toFile); + } catch (IOException e) { + throw new IllegalStateException("Can not zip directory " + dir + " to file " + toFile, e); + } + } + + /** + * Creates specified directory if it does not exist yet and any non existing parent. + * + * @throws IllegalStateException if specified File exists but is not a directory + * @throws IllegalStateException if directory creation failed + */ + public void createDir(File dir) { + Path dirPath = requireNonNull(dir, "dir can not be null").toPath(); + if (dirPath.toFile().exists()) { + checkState(dirPath.toFile().isDirectory(), "%s is not a directory", dirPath); + } else { + try { + createDirectories(dirPath); + } catch (IOException e) { + throw new IllegalStateException(format("Failed to create directory %s", dirPath), e); + } + } + } + + private static void checkOrThrowIOE(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) throws IOException { + if (!expression) { + throw new IOException(format(errorMessageTemplate, errorMessageArgs)); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Protobuf2.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Protobuf2.java new file mode 100644 index 00000000000..ec1ef74aa24 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/util/Protobuf2.java @@ -0,0 +1,71 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.util; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.Parser; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import javax.annotation.CheckForNull; + +public class Protobuf2 { + + public static final Protobuf2 PROTOBUF2 = new Protobuf2(); + + private Protobuf2() { + } + + public Protobuf2 writeTo(Message msg, OutputStream output) { + try { + msg.writeTo(output); + } catch (IOException e) { + throw new IllegalStateException("Can not write message " + msg, e); + } + return this; + } + + public Protobuf2 writeDelimitedTo(Message msg, OutputStream output) { + try { + msg.writeDelimitedTo(output); + } catch (IOException e) { + throw new IllegalStateException("Can not write message " + msg, e); + } + return this; + } + + public MSG parseFrom(Parser parser, InputStream input) { + try { + return parser.parseFrom(input); + } catch (InvalidProtocolBufferException e) { + throw new IllegalStateException("Can not parse message", e); + } + } + + @CheckForNull + public MSG parseDelimitedFrom(Parser parser, InputStream input) { + try { + return parser.parseDelimitedFrom(input); + } catch (InvalidProtocolBufferException e) { + throw new IllegalStateException("Can not parse message", e); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto b/server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto new file mode 100644 index 00000000000..170f8a48b6e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto @@ -0,0 +1,218 @@ +syntax = "proto3"; + +package org.sonar.ce.task.projectexport.projectdump; + +option java_package = "com.sonarsource.governance.projectdump.protobuf"; +option optimize_for = SPEED; + +// single message in metadata.pb +message Metadata { + string project_key = 1; + // all component uuids are kept in the target environment, except for the project + // as it has already been provisioned with a different uuid. + string project_uuid = 2; + string sonarqube_version = 3; + int64 dump_date = 4; +} + +// Stream of components stored in file components.pb, including project. +// Components are ordered by id, so that parents are +// always located before children. +message Component { + int64 ref = 1; + string key = 2; + string name = 3; + string description = 4; + string scope = 5; + string qualifier = 6; + string language = 7; + string long_name = 8; + string path = 9; + string uuid = 10; + string uuid_path = 12; + string module_uuid = 13; + string module_uuid_path = 14; + string deprecated_key = 15; + string project_uuid = 16; + string main_branch_project_uuid = 17; +} + +message Branch { + int64 ref = 1; + string uuid = 2; + string project_uuid = 3; + string kee = 4; + string branch_type = 5; + string merge_branch_uuid = 6; +} + +// Stream of analyses stored in file analyses.pb, including project. +// Only analyses with status 'P' (processed) are exported. They are ordered +// by build_date, so that parents are always located before children. +message Analysis { + int64 component_ref = 2; + int64 date = 3; + string projectVersion = 4; + string period1_mode = 5; + string period1_param = 6; + int64 period1_date = 7; + string uuid = 8; + string buildString = 9; +} + +message Metric { + int32 ref = 1; + string key = 2; + string name = 3; +} + +message Measure { + int64 component_ref = 1; + string analysis_uuid = 2; + int32 metric_ref = 3; + DoubleValue double_value = 4; + string text_value = 5; + string alert_status = 6; + string alert_text = 7; + DoubleValue variation1 = 8; +} + +message LiveMeasure { + int64 component_ref = 1; + int32 metric_ref = 2; + DoubleValue double_value = 3; + string text_value = 4; + DoubleValue variation = 5; +} + +message DoubleValue { + double value = 1; +} + +// Stream of issues stored in file issues.pb +// only issues which status is not 'CLOSED' are exported +// rule_ref field refers to the ref of the Rule message +message Issue { + string uuid = 1; + int64 component_ref = 2; + int32 type = 4; + + string message = 5; + + int32 line = 6; + string checksum = 7; + bytes locations = 8; + + string status = 9; + string resolution = 10; + string severity = 11; + bool manual_severity = 12; + double gap = 13; + int64 effort = 14; + + string assignee = 15; + string author = 16; + + string tags = 17; + string attributes = 18; + // issue dates + int64 issue_created_at = 19; + int64 issue_updated_at = 20; + int64 issue_closed_at = 21; + string project_uuid = 22; + string rule_ref = 23; +} + +// Stream of issues changelog stored in file issues_changelog.pb +// only changes of issues present in the dump are exported +message IssueChange { + string key = 1; + string issue_uuid = 2; + string project_uuid = 3; + string change_type = 4; + string change_data = 5; + string user_uuid = 6; + int64 created_at = 7; +} + +// Stream of rules is stored in file rules.pb +// field ref is a unique rule identifier, internal to the dump. +message Rule { + string key = 2; + string repository = 3; + string ref = 4; +} + +// Stream of ad hoc rules is stored in file ad_hoc_rules.pb +// field ref is a unique rule identifier, internal to the dump, starting with 1. +message AdHocRule { + + message RuleMetadata { + string ad_hoc_name = 2; + string ad_hoc_description = 3; + string ad_hoc_severity = 4; + int32 ad_hoc_type = 5; + } + + string ref = 10; + string plugin_key = 2; + string plugin_rule_key = 3; + string plugin_name = 4; + string name = 5; + string status = 6; + int32 type = 7; + string scope = 8; + RuleMetadata metadata = 9; +} + +// stream of messages in settings.pb +message Setting { + string key = 1; + string value = 2; + int64 component_ref = 3; +} + +// links specified on the project, and only on the project (not on modules +// nor other components) +message Link { + string type = 1; + string name = 2; + string href = 3; + string uuid = 4; + int64 component_ref = 5; +} + +message Event { + string name = 1; + string analysis_uuid = 2; + int64 component_ref = 3; + string category = 4; + string description = 5; + string data = 6; + int64 date = 7; + string uuid = 8; +} + +message Plugin { + string key = 1; + string name = 2; + string version = 3; +} + +// Stream of line hashes of each Component of type FILE in the dump +// only lines hashes of source file of type SOURCE are exported +message LineHashes { + int64 component_ref = 1; + string hashes = 2; + string project_uuid = 3; +} + +message NewCodePeriod { + string uuid = 1; + string project_uuid = 2; + string branch_uuid = 3; + string type = 4; + string value = 5; + int64 created_at = 6; + int64 updated_at = 7; +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportComputationStepsTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportComputationStepsTest.java new file mode 100644 index 00000000000..94377d5169e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportComputationStepsTest.java @@ -0,0 +1,73 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport; + +import com.google.common.collect.Lists; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.container.TaskContainerImpl; +import org.sonar.ce.task.projectanalysis.step.ComplexityMeasuresStep; +import org.sonar.ce.task.projectexport.steps.LoadProjectStep; +import org.sonar.core.platform.ComponentContainer; + +import static com.google.common.collect.ImmutableList.copyOf; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +public class ProjectExportComputationStepsTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private TaskContainer container = mock(TaskContainer.class); + private ProjectExportComputationSteps underTest = new ProjectExportComputationSteps(container); + + @Test + public void count_step_classes() { + assertThat(copyOf(underTest.orderedStepClasses()).size()).isEqualTo(20); + } + + @Test + public void instances_throws_ISE_if_steps_do_not_exist_in_container() { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Component not found: " + LoadProjectStep.class); + + copyOf(underTest.instances()); + } + + @Test + public void instances_throws_ISE_if_container_does_not_have_second_step() { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Component not found: class org.sonar.ce.task.projectexport.steps.LoadProjectStep"); + + final ComplexityMeasuresStep reportExtractionStep = mock(ComplexityMeasuresStep.class); + ComponentContainer componentContainer = new ComponentContainer() { + { + addSingleton(reportExtractionStep); + } + }; + TaskContainerImpl computeEngineContainer = new TaskContainerImpl(componentContainer, container -> { + // do nothing + }); + + Lists.newArrayList(new ProjectExportComputationSteps(computeEngineContainer).instances()); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulatorTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulatorTest.java new file mode 100644 index 00000000000..fa107347432 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/ProjectExportContainerPopulatorTest.java @@ -0,0 +1,105 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport; + +import com.google.common.base.Predicate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.junit.Test; +import org.picocontainer.PicoContainer; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.setting.SettingsLoader; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.core.platform.ComponentContainer; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ProjectExportContainerPopulatorTest { + + private static final int COMPONENTS_BY_DEFAULT_IN_CONTAINER = 2; + + private final ProjectDescriptor descriptor = new ProjectDescriptor("project_uuid", "project_key", "Project Name"); + private final ProjectExportContainerPopulator underTest = new ProjectExportContainerPopulator(descriptor); + + @Test + public void test_populateContainer() { + RecorderTaskContainer container = new RecorderTaskContainer(); + underTest.populateContainer(container); + assertThat(container.addedComponents) + .hasSize(COMPONENTS_BY_DEFAULT_IN_CONTAINER + 8) + .contains(descriptor, SettingsLoader.class); + } + + private static class RecorderTaskContainer implements TaskContainer { + private final List addedComponents = new ArrayList<>(); + + @Override + public ComponentContainer add(Object... objects) { + addedComponents.addAll(Arrays.asList(objects)); + // not used anyway + return null; + } + + @Override + public ComponentContainer addSingletons(Iterable components) { + List filteredComponents = StreamSupport.stream(components.spliterator(), false) + .filter((Predicate) input -> !(input instanceof Class) || !ComputationStep.class.isAssignableFrom((Class) input)) + .collect(Collectors.toList()); + + addedComponents.addAll(filteredComponents); + // not used anyway + return null; + } + + @Override + public ComponentContainer getParent() { + throw new UnsupportedOperationException("getParent is not implemented"); + } + + @Override + public void bootup() { + throw new UnsupportedOperationException("bootup is not implemented"); + } + + @Override + public void close() { + throw new UnsupportedOperationException("close is not implemented"); + } + + @Override + public PicoContainer getPicoContainer() { + throw new UnsupportedOperationException("getParent is not implemented"); + } + + @Override + public T getComponentByType(Class type) { + throw new UnsupportedOperationException("getParent is not implemented"); + } + + @Override + public List getComponentsByType(final Class type) { + throw new UnsupportedOperationException("getParent is not implemented"); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStepTest.java new file mode 100644 index 00000000000..34d6fbf1fac --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/analysis/ExportAnalysesStepTest.java @@ -0,0 +1,223 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.analysis; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.List; +import javax.annotation.Nullable; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.SnapshotDto; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.apache.commons.lang.StringUtils.defaultString; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; +import static org.sonar.db.component.ComponentDto.UUID_PATH_SEPARATOR; + +@RunWith(DataProviderRunner.class) +public class ExportAnalysesStepTest { + + private static final String PROJECT_UUID = "PROJECT_UUID"; + private static final ComponentDto PROJECT = new ComponentDto() + // no id yet + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.PROJECT) + .setDbKey("the_project") + .setName("The Project") + .setDescription("The project description") + .setEnabled(true) + .setUuid(PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid(PROJECT_UUID) + .setModuleUuid(null) + .setModuleUuidPath("." + PROJECT_UUID + ".") + .setProjectUuid(PROJECT_UUID); + + private static final String MODULE_UUID = "MODULE_UUID"; + private static final String MODULE_UUID_PATH = UUID_PATH_OF_ROOT + UUID_PATH_SEPARATOR + MODULE_UUID; + private static final ComponentDto MODULE = new ComponentDto() + // no id yet + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.MODULE) + .setDbKey("the_module") + .setName("The Module") + .setDescription("description of module") + .setEnabled(true) + .setUuid(MODULE_UUID) + .setUuidPath(MODULE_UUID_PATH) + .setRootUuid(MODULE_UUID) + .setModuleUuid(PROJECT_UUID) + .setModuleUuidPath("." + PROJECT_UUID + ".MODULE_UUID.") + .setProjectUuid(PROJECT_UUID); + + private static final String FILE_UUID = "FILE_UUID"; + private static final ComponentDto FILE = new ComponentDto() + // no id yet + .setScope(Scopes.FILE) + .setQualifier(Qualifiers.FILE) + .setDbKey("the_file") + .setName("The File") + .setUuid(FILE_UUID) + .setUuidPath(MODULE_UUID_PATH + UUID_PATH_SEPARATOR + FILE_UUID) + .setRootUuid(MODULE_UUID) + .setEnabled(true) + .setModuleUuid(MODULE_UUID) + .setModuleUuidPath("." + PROJECT_UUID + ".MODULE_UUID.") + .setProjectUuid(PROJECT_UUID); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + @Rule + public LogTester logTester = new LogTester(); + + private final ComponentRepositoryImpl componentRepository = new ComponentRepositoryImpl(); + private final FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private final ProjectHolder projectHolder = mock(ProjectHolder.class); + private final ExportAnalysesStep underTest = new ExportAnalysesStep(dbTester.getDbClient(), projectHolder, componentRepository, dumpWriter); + + @Before + public void setUp() { + ComponentDto projectDto = dbTester.components().insertPublicProject(PROJECT); + componentRepository.register(1, projectDto.uuid(), false); + dbTester.getDbClient().componentDao().insert(dbTester.getSession(), MODULE, FILE); + dbTester.commit(); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(projectDto)); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isEqualTo("Export analyses"); + } + + @Test + @UseDataProvider("versionAndBuildStringCombinations") + public void export_analyses(@Nullable String version, @Nullable String buildString) { + SnapshotDto firstAnalysis = newAnalysis("U_1", 1_450_000_000_000L, PROJECT.uuid(), "1.0", false, "1.0.2.3", 1_450_000_000_000L); + SnapshotDto secondAnalysis = newAnalysis("U_4", 1_460_000_000_000L, PROJECT.uuid(), "1.1", true, "1.1.3.4", 1_460_000_000_000L); + SnapshotDto thirdAnalysis = newAnalysis("U_7", 1_460_000_000_000L, PROJECT.uuid(), version, true, buildString, 1_470_000_000_000L); + dbTester.getDbClient().snapshotDao().insert(dbTester.getSession(), firstAnalysis, secondAnalysis, thirdAnalysis); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("3 analyses exported"); + List analyses = dumpWriter.getWrittenMessagesOf(DumpElement.ANALYSES); + assertThat(analyses).hasSize(3); + assertAnalysis(analyses.get(0), PROJECT, firstAnalysis); + assertAnalysis(analyses.get(1), PROJECT, secondAnalysis); + assertAnalysis(analyses.get(2), PROJECT, thirdAnalysis); + } + + @DataProvider + public static Object[][] versionAndBuildStringCombinations() { + String version = randomAlphabetic(7); + String buildString = randomAlphabetic(12); + return new Object[][] { + {null, null}, + {version, null}, + {null, buildString}, + {version, buildString}, + {"", ""}, + {version, ""}, + {"", buildString}, + }; + } + + @Test + public void export_analyses_by_ordering_by_technical_creation_date() { + SnapshotDto firstAnalysis = newAnalysis("U_1", 1_450_000_000_000L, PROJECT.uuid(), "1.0", false, "1.0.2.3", 3_000_000_000_000L); + SnapshotDto secondAnalysis = newAnalysis("U_4", 1_460_000_000_000L, PROJECT.uuid(), "1.1", true, "1.1.3.4", 1_000_000_000_000L); + SnapshotDto thirdAnalysis = newAnalysis("U_7", 1_460_500_000_000L, PROJECT.uuid(), null, true, null, 2_000_000_000_000L); + dbTester.getDbClient().snapshotDao().insert(dbTester.getSession(), firstAnalysis, secondAnalysis, thirdAnalysis); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + List analyses = dumpWriter.getWrittenMessagesOf(DumpElement.ANALYSES); + assertAnalysis(analyses.get(0), PROJECT, secondAnalysis); + assertAnalysis(analyses.get(1), PROJECT, thirdAnalysis); + assertAnalysis(analyses.get(2), PROJECT, firstAnalysis); + } + + @Test + public void export_provisioned_projects_without_any_analyses() { + underTest.execute(new TestComputationStepContext()); + + List analyses = dumpWriter.getWrittenMessagesOf(DumpElement.ANALYSES); + assertThat(analyses).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 analyses exported"); + } + + @Test + public void throws_ISE_if_error() { + SnapshotDto firstAnalysis = newAnalysis("U_1", 1_450_000_000_000L, PROJECT.uuid(), "1.0", false, "1.0.2.3", 1); + SnapshotDto secondAnalysis = newAnalysis("U_4", 1_460_000_000_000L, PROJECT.uuid(), "1.1", true, "1.1.3.4", 2); + dbTester.getDbClient().snapshotDao().insert(dbTester.getSession(), firstAnalysis, secondAnalysis); + dbTester.commit(); + dumpWriter.failIfMoreThan(1, DumpElement.ANALYSES); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Analysis Export failed after processing 1 analyses successfully"); + underTest.execute(new TestComputationStepContext()); + } + + private static SnapshotDto newAnalysis(String uuid, long date, String componentUuid, @Nullable String version, boolean isLast, @Nullable String buildString, long buildDate) { + return new SnapshotDto() + .setUuid(uuid) + .setCreatedAt(date) + .setComponentUuid(componentUuid) + .setProjectVersion(version) + .setBuildString(buildString) + .setLast(isLast) + .setStatus(SnapshotDto.STATUS_PROCESSED) + .setBuildDate(buildDate); + } + + private static void assertAnalysis(ProjectDump.Analysis analysis, ComponentDto component, SnapshotDto dto) { + assertThat(analysis.getUuid()).isEqualTo(dto.getUuid()); + assertThat(analysis.getComponentRef()).isEqualTo(1); + assertThat(analysis.getDate()).isEqualTo(dto.getCreatedAt()); + assertThat(analysis.getProjectVersion()).isEqualTo(defaultString(dto.getProjectVersion())); + assertThat(analysis.getBuildString()).isEqualTo(defaultString(dto.getBuildString())); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStepTest.java new file mode 100644 index 00000000000..d31477b1754 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/branches/ExportBranchesStepTest.java @@ -0,0 +1,161 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.branches; + +import com.google.common.collect.ImmutableList; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.apache.commons.lang.time.DateUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.project.ProjectExportMapper; + +import static java.util.stream.Collectors.toMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; + +public class ExportBranchesStepTest { + private static final String PROJECT_UUID = "PROJECT_UUID"; + private static final ComponentDto PROJECT = new ComponentDto() + // no id yet + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.PROJECT) + .setDbKey("the_project") + .setName("The Project") + .setDescription("The project description") + .setEnabled(true) + .setUuid(PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid(PROJECT_UUID) + .setModuleUuid(null) + .setModuleUuidPath("." + PROJECT_UUID + ".") + .setProjectUuid(PROJECT_UUID); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public DbTester dbTester = DbTester.createWithExtensionMappers(System2.INSTANCE, ProjectExportMapper.class); + @Rule + public LogTester logTester = new LogTester(); + + private final ProjectHolder projectHolder = mock(ProjectHolder.class); + private final FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private final ExportBranchesStep underTest = new ExportBranchesStep(dumpWriter, dbTester.getDbClient(), projectHolder); + private final List branches = ImmutableList.of( + new BranchDto() + .setBranchType(BranchType.BRANCH) + .setProjectUuid(PROJECT_UUID) + .setKey("branch-1") + .setUuid("branch-1-uuid") + .setMergeBranchUuid("master") + .setExcludeFromPurge(true), + new BranchDto() + .setBranchType(BranchType.PULL_REQUEST) + .setProjectUuid(PROJECT_UUID) + .setKey("branch-3") + .setUuid("branch-3-uuid") + .setMergeBranchUuid("master"), + new BranchDto() + .setBranchType(BranchType.BRANCH) + .setProjectUuid(PROJECT_UUID) + .setKey("branch-4") + .setUuid("branch-4-uuid") + .setMergeBranchUuid("branch-1-uuid"), + new BranchDto() + .setBranchType(BranchType.BRANCH) + .setProjectUuid(PROJECT_UUID) + .setKey("branch-5") + .setUuid("branch-5-uuid") + .setMergeBranchUuid("master") + .setExcludeFromPurge(true)); + + @Before + public void setUp() { + Date createdAt = new Date(); + ComponentDto projectDto = dbTester.components().insertPublicProject(PROJECT).setCreatedAt(createdAt); + for (BranchDto branch : branches) { + createdAt = DateUtils.addMinutes(createdAt, 10); + dbTester.components().insertProjectBranch(PROJECT, branch).setCreatedAt(createdAt); + } + dbTester.commit(); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(projectDto)); + } + + @Test + public void export_branches() { + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("3 branches exported"); + Map branches = dumpWriter.getWrittenMessagesOf(DumpElement.BRANCHES) + .stream() + .collect(toMap(ProjectDump.Branch::getUuid, Function.identity())); + assertThat(branches).hasSize(3); + ProjectDump.Branch masterBranch = branches.get(PROJECT_UUID); + assertThat(masterBranch).isNotNull(); + assertThat(masterBranch.getKee()).isEqualTo("master"); + assertThat(masterBranch.getProjectUuid()).isEqualTo(PROJECT_UUID); + assertThat(masterBranch.getMergeBranchUuid()).isEmpty(); + assertThat(masterBranch.getBranchType()).isEqualTo("BRANCH"); + ProjectDump.Branch branch1 = branches.get("branch-1-uuid"); + assertThat(branch1.getKee()).isEqualTo("branch-1"); + assertThat(branch1.getProjectUuid()).isEqualTo(PROJECT_UUID); + assertThat(branch1.getMergeBranchUuid()).isEqualTo("master"); + assertThat(branch1.getBranchType()).isEqualTo("BRANCH"); + ProjectDump.Branch branch5 = branches.get("branch-5-uuid"); + assertThat(branch5.getKee()).isEqualTo("branch-5"); + assertThat(branch5.getProjectUuid()).isEqualTo(PROJECT_UUID); + assertThat(branch5.getMergeBranchUuid()).isEqualTo("master"); + assertThat(branch5.getBranchType()).isEqualTo("BRANCH"); + } + + @Test + public void throws_ISE_if_error() { + dumpWriter.failIfMoreThan(1, DumpElement.BRANCHES); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Branch export failed after processing 1 branch(es) successfully"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isEqualTo("Export branches"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImplTest.java new file mode 100644 index 00000000000..317d0faf90b --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ComponentRepositoryImplTest.java @@ -0,0 +1,119 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.component; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ComponentRepositoryImplTest { + private static final int SOME_REF = 121; + private static final String SOME_UUID = "uuid"; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private ComponentRepositoryImpl underTest = new ComponentRepositoryImpl(); + + @Test + public void register_throws_NPE_if_uuid_is_null() { + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("uuid can not be null"); + + underTest.register(SOME_REF, null, true); + } + + @Test + public void register_throws_IAE_same_uuid_added_with_different_refs() { + underTest.register(SOME_REF, SOME_UUID, true); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Uuid '" + SOME_UUID + "' already registered under ref '" + SOME_REF + "' in repository"); + + underTest.register(946512, SOME_UUID, true); + } + + @Test + public void register_throws_IAE_same_uuid_added_with_as_file() { + underTest.register(SOME_REF, SOME_UUID, true); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Uuid '" + SOME_UUID + "' already registered but as a File"); + + underTest.register(SOME_REF, SOME_UUID, false); + } + + @Test + public void register_throws_IAE_same_uuid_added_with_as_not_file() { + underTest.register(SOME_REF, SOME_UUID, false); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Uuid '" + SOME_UUID + "' already registered but not as a File"); + + underTest.register(SOME_REF, SOME_UUID, true); + } + + @Test + public void getRef_throws_NPE_if_uuid_is_null() { + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("uuid can not be null"); + + underTest.getRef(null); + } + + @Test + public void getRef_throws_ISE_if_uuid_not_in_repository() { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("No reference registered in the repository for uuid '" + SOME_UUID + "'"); + + underTest.getRef(SOME_UUID); + } + + @Test + public void getRef_returns_ref_added_with_register_when_file() { + underTest.register(SOME_REF, SOME_UUID, true); + + assertThat(underTest.getRef(SOME_UUID)).isEqualTo(SOME_REF); + } + + @Test + public void getRef_returns_ref_added_with_register_when_not_file() { + underTest.register(SOME_REF, SOME_UUID, false); + + assertThat(underTest.getRef(SOME_UUID)).isEqualTo(SOME_REF); + } + + @Test + public void getFileUuids_returns_empty_when_repository_is_empty() { + assertThat(underTest.getFileUuids()).isEmpty(); + } + + @Test + public void getFileUuids_returns_uuids_of_only_components_added_with_file_flag_is_true() { + underTest.register(SOME_REF, "file id 1", true); + underTest.register(546, SOME_UUID, false); + underTest.register(987, "file id 2", true); + underTest.register(123, "not file id", false); + + assertThat(underTest.getFileUuids()).containsOnly("file id 1", "file id 2"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ExportComponentsStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ExportComponentsStepTest.java new file mode 100644 index 00000000000..7a0bade945f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/component/ExportComponentsStepTest.java @@ -0,0 +1,176 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.component; + +import com.google.common.collect.ImmutableSet; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.Date; +import java.util.List; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; +import static org.sonar.db.component.ComponentDto.UUID_PATH_SEPARATOR; + +public class ExportComponentsStepTest { + + private static final String PROJECT_UUID = "PROJECT_UUID"; + private static final ComponentDto PROJECT = new ComponentDto() + // no id yet + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.PROJECT) + .setDbKey("the_project") + .setName("The Project") + .setDescription("The project description") + .setEnabled(true) + .setUuid(PROJECT_UUID) + .setRootUuid(PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setModuleUuid(null) + .setModuleUuidPath("." + PROJECT_UUID + ".") + .setCreatedAt(new Date(1596749115856L)) + .setProjectUuid(PROJECT_UUID); + + private static final String MODULE_UUID = "MODULE_UUID"; + private static final String MODULE_UUID_PATH = UUID_PATH_OF_ROOT + MODULE_UUID + UUID_PATH_SEPARATOR; + private static final ComponentDto MODULE = new ComponentDto() + // no id yet + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.MODULE) + .setDbKey("the_module") + .setName("The Module") + .setDescription("description of module") + .setEnabled(true) + .setUuid(MODULE_UUID) + .setRootUuid(PROJECT_UUID) + .setUuidPath(MODULE_UUID_PATH) + .setModuleUuid(PROJECT_UUID) + .setModuleUuidPath("." + PROJECT_UUID + ".MODULE_UUID.") + .setCreatedAt(new Date(1596749132539L)) + .setProjectUuid(PROJECT_UUID); + + private static final String FILE_UUID = "FILE_UUID"; + private static final String FILE_UUID_PATH = MODULE_UUID_PATH + FILE_UUID + UUID_PATH_SEPARATOR; + private static final ComponentDto FILE = new ComponentDto() + // no id yet + .setScope(Scopes.FILE) + .setQualifier(Qualifiers.FILE) + .setDbKey("the_file") + .setName("The File") + .setUuid(FILE_UUID) + .setRootUuid(MODULE_UUID) + .setUuidPath(FILE_UUID_PATH) + .setEnabled(true) + .setModuleUuid(MODULE_UUID) + .setModuleUuidPath("." + PROJECT_UUID + ".MODULE_UUID.") + .setCreatedAt(new Date(1596749148406L)) + .setProjectUuid(PROJECT_UUID); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public LogTester logTester = new LogTester(); + + private final FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private final ProjectHolder projectHolder = mock(ProjectHolder.class); + private final MutableComponentRepository componentRepository = new ComponentRepositoryImpl(); + private final ExportComponentsStep underTest = new ExportComponentsStep(dbTester.getDbClient(), projectHolder, componentRepository, dumpWriter); + + @After + public void tearDown() { + dbTester.getSession().close(); + } + + @Test + public void export_components_including_project() { + dbTester.components().insertPublicProject(PROJECT); + dbTester.getDbClient().componentDao().insert(dbTester.getSession(), MODULE, FILE); + dbTester.commit(); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(PROJECT)); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("3 components exported"); + List components = dumpWriter.getWrittenMessagesOf(DumpElement.COMPONENTS); + assertThat(components).extracting(ProjectDump.Component::getQualifier, ProjectDump.Component::getUuid, ProjectDump.Component::getUuidPath) + .containsExactlyInAnyOrder( + tuple(Qualifiers.FILE, FILE_UUID, FILE_UUID_PATH), + tuple(Qualifiers.MODULE, MODULE_UUID, MODULE_UUID_PATH), + tuple(Qualifiers.PROJECT, PROJECT_UUID, UUID_PATH_OF_ROOT)); + } + + @Test + public void execute_register_all_components_uuids_as_their_id_in_ComponentRepository() { + dbTester.components().insertPublicProject(PROJECT); + dbTester.getDbClient().componentDao().insert(dbTester.getSession(), MODULE, FILE); + dbTester.commit(); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(PROJECT)); + + underTest.execute(new TestComputationStepContext()); + + assertThat(ImmutableSet.of( + componentRepository.getRef(PROJECT.uuid()), + componentRepository.getRef(MODULE.uuid()), + componentRepository.getRef(FILE.uuid()))).containsExactlyInAnyOrder(1L, 2L, 3L); + } + + @Test + public void throws_ISE_if_error() { + dbTester.components().insertPublicProject(PROJECT); + dbTester.getDbClient().componentDao().insert(dbTester.getSession(), MODULE, FILE); + dbTester.commit(); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(PROJECT)); + dumpWriter.failIfMoreThan(1, DumpElement.COMPONENTS); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Component Export failed after processing 1 components successfully"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isEqualTo("Export components"); + } + + @Test + public void name() { + System.err.println(new Date(1313272860000L)); + + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStepTest.java new file mode 100644 index 00000000000..9914c2edaf6 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/file/ExportLineHashesStepTest.java @@ -0,0 +1,195 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.file; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.projectexport.component.MutableComponentRepository; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.core.util.Uuids; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.MyBatis; +import org.sonar.db.source.FileSourceDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +public class ExportLineHashesStepTest { + private static final String PROJECT_MASTER_UUID = "project uuid"; + private static final String PROJECT_BRANCH_UUID = "branch-uuid"; + private static final String FILE_UUID = "file uuid"; + private static final String FILE_UUID_2 = "file-2-uuid"; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + @Rule + public LogTester logTester = new LogTester(); + + private DbClient dbClient = dbTester.getDbClient(); + private DbSession dbSession = dbClient.openSession(false); + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private MutableComponentRepository componentRepository = new ComponentRepositoryImpl(); + + private ExportLineHashesStep underTest = new ExportLineHashesStep(dbClient, dumpWriter, componentRepository); + + @After + public void tearDown() { + dbSession.close(); + } + + @Test + public void getDescription_is_set() { + assertThat(underTest.getDescription()).isEqualTo("Export line hashes"); + } + + @Test + public void execute_does_not_create_a_session_when_there_is_no_file_in_ComponentRepository() { + DbClient spy = spy(dbClient); + new ExportLineHashesStep(spy, dumpWriter, componentRepository) + .execute(new TestComputationStepContext()); + + verifyNoInteractions(spy); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.LINES_HASHES)).isEmpty(); + } + + @Test + public void execute_relies_only_on_file_uuid_and_does_not_check_project_uuid() { + componentRepository.register(1, FILE_UUID, true); + + insertFileSource(createDto(FILE_UUID, "blabla", "A")); + insertFileSource(createDto(FILE_UUID_2, "blabla", "B")); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.LINES_HASHES)).hasSize(1); + } + + @Test + public void execute_maps_ref_of_component_and_hashes_from_fileSources() { + int fileRef = 984615; + componentRepository.register(fileRef, FILE_UUID, true); + FileSourceDto dto = createDto(FILE_UUID, PROJECT_MASTER_UUID, "B"); + insertFileSource(dto); + + underTest.execute(new TestComputationStepContext()); + + List messages = dumpWriter.getWrittenMessagesOf(DumpElement.LINES_HASHES); + assertThat(messages).hasSize(1); + ProjectDump.LineHashes lineHashes = messages.iterator().next(); + assertThat(lineHashes.getHashes()).isEqualTo(dto.getRawLineHashes()); + assertThat(lineHashes.getComponentRef()).isEqualTo(fileRef); + } + + @Test + public void execute_does_one_SQL_request_by_1000_items_per_IN_clause() { + for (int i = 0; i < 2500; i++) { + componentRepository.register(i, "uuid_" + i, true); + } + + DbClient spyDbClient = spy(dbClient); + MyBatis spyMyBatis = spy(dbClient.getMyBatis()); + when(spyDbClient.getMyBatis()).thenReturn(spyMyBatis); + ArgumentCaptor stringCaptor = ArgumentCaptor.forClass(String.class); + doCallRealMethod().when(spyMyBatis).newScrollingSelectStatement(any(DbSession.class), stringCaptor.capture()); + + new ExportLineHashesStep(spyDbClient, dumpWriter, componentRepository) + .execute(new TestComputationStepContext()); + + List statements = stringCaptor.getAllValues(); + assertThat(statements).hasSize(3); + + assertThat(statements.get(0).split("\\?")).hasSize(1001); + assertThat(statements.get(1).split("\\?")).hasSize(1001); + assertThat(statements.get(2).split("\\?")).hasSize(501); + } + + @Test + public void execute_exports_lines_hashes_of_file_sources() { + componentRepository.register(1, FILE_UUID, true); + insertFileSource(FILE_UUID, "A"); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.LINES_HASHES)) + .extracting(ProjectDump.LineHashes::getHashes) + .containsOnly("A"); + } + + @Test + public void execute_logs_number_of_filesource_exported_and_export_by_order_of_id() { + componentRepository.register(1, FILE_UUID, true); + componentRepository.register(2, "file-2", true); + componentRepository.register(3, "file-3", true); + componentRepository.register(4, "file-4", true); + + insertFileSource(createDto(FILE_UUID, PROJECT_MASTER_UUID, "A")); + insertFileSource(createDto("file-2", PROJECT_MASTER_UUID, "C")); + insertFileSource(createDto("file-3", PROJECT_MASTER_UUID, "D")); + insertFileSource(createDto("file-4", PROJECT_BRANCH_UUID, "E")); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.LINES_HASHES)) + .extracting(ProjectDump.LineHashes::getHashes) + .containsExactly("A", "C", "D", "E"); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).containsExactly("Lines hashes of 4 files exported"); + } + + private FileSourceDto insertFileSource(String fileUuid, String hashes) { + FileSourceDto dto = createDto(fileUuid, PROJECT_MASTER_UUID, hashes); + return insertFileSource(dto); + } + + private FileSourceDto insertFileSource(FileSourceDto dto) { + dbClient.fileSourceDao().insert(dbSession, dto); + dbSession.commit(); + return dto; + } + + private FileSourceDto createDto(String fileUuid, String componentUuid, String hashes) { + FileSourceDto fileSourceDto = new FileSourceDto() + .setUuid(Uuids.createFast()) + .setFileUuid(fileUuid) + .setProjectUuid(componentUuid); + fileSourceDto.setRawLineHashes(hashes); + return fileSourceDto; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStepTest.java new file mode 100644 index 00000000000..4a1705280b6 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesChangelogStepTest.java @@ -0,0 +1,277 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.issue; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import javax.annotation.Nullable; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.core.util.Uuids; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.issue.IssueChangeDto; +import org.sonar.db.issue.IssueDto; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.api.issue.Issue.STATUS_CLOSED; +import static org.sonar.api.issue.Issue.STATUS_CONFIRMED; +import static org.sonar.api.issue.Issue.STATUS_OPEN; +import static org.sonar.api.issue.Issue.STATUS_REOPENED; +import static org.sonar.api.issue.Issue.STATUS_RESOLVED; + +public class ExportIssuesChangelogStepTest { + private static final String PROJECT_UUID = "project uuid"; + private static final String ISSUE_OPEN_UUID = "issue 1 uuid"; + private static final String ISSUE_CONFIRMED_UUID = "issue 2 uuid"; + private static final String ISSUE_REOPENED_UUID = "issue 3 uuid"; + private static final String ISSUE_RESOLVED_UUID = "issue 4 uuid"; + private static final String ISSUE_CLOSED_UUID = "issue closed uuid"; + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + @Rule + public LogTester logTester = new LogTester(); + + private final DbClient dbClient = dbTester.getDbClient(); + private final DbSession dbSession = dbClient.openSession(false); + private final ProjectHolder projectHolder = mock(ProjectHolder.class); + private final FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private final ExportIssuesChangelogStep underTest = new ExportIssuesChangelogStep(dbClient, projectHolder, dumpWriter); + + private int issueChangeUuidGenerator = 0; + + @Before + public void setUp() { + ComponentDto projectDto = dbTester.components().insertPublicProject(p -> p.setUuid(PROJECT_UUID)); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(projectDto)); + when(projectHolder.branches()).thenReturn(newArrayList( + new BranchDto().setBranchType(BranchType.BRANCH).setKey("master").setProjectUuid(PROJECT_UUID).setUuid(PROJECT_UUID))); + + insertIssue(PROJECT_UUID, ISSUE_OPEN_UUID, STATUS_OPEN); + insertIssue(PROJECT_UUID, ISSUE_CONFIRMED_UUID, STATUS_CONFIRMED); + insertIssue(PROJECT_UUID, ISSUE_REOPENED_UUID, STATUS_REOPENED); + insertIssue(PROJECT_UUID, ISSUE_RESOLVED_UUID, STATUS_RESOLVED); + insertIssue(PROJECT_UUID, ISSUE_CLOSED_UUID, STATUS_CLOSED); + } + + @After + public void tearDown() { + dbSession.close(); + } + + @Test + public void getDescription_is_set() { + assertThat(underTest.getDescription()).isEqualTo("Export issues changelog"); + } + + @Test + public void execute_writes_now_RuleChange_is_db_is_empty() { + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES_CHANGELOG)).isEmpty(); + } + + @Test + public void execute_writes_entries_of_issues_of_any_type_but_CLOSED() { + long createdAt = 1_999_993L; + String[] expectedKeys = new String[] { + insertIssueChange(ISSUE_OPEN_UUID, createdAt).getKey(), + insertIssueChange(ISSUE_CONFIRMED_UUID, createdAt + 1).getKey(), + insertIssueChange(ISSUE_REOPENED_UUID, createdAt + 2).getKey(), + insertIssueChange(ISSUE_RESOLVED_UUID, createdAt + 3).getKey() + }; + insertIssueChange(ISSUE_CLOSED_UUID, createdAt + 4); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES_CHANGELOG)) + .extracting(ProjectDump.IssueChange::getKey).containsExactly(expectedKeys); + } + + @Test + public void execute_writes_only_entries_of_current_project() { + String issueUuid = "issue uuid"; + insertIssue("other project uuid", issueUuid, STATUS_OPEN); + insertIssueChange(issueUuid); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES_CHANGELOG)).isEmpty(); + } + + @Test + public void execute_maps_all_fields_to_protobuf() { + IssueChangeDto issueChangeDto = new IssueChangeDto() + .setUuid("uuid") + .setKey("key") + .setIssueKey(ISSUE_OPEN_UUID) + .setChangeData("change data") + .setChangeType("change type") + .setUserUuid("user_uuid") + .setIssueChangeCreationDate(454135L) + .setProjectUuid(PROJECT_UUID); + insertIssueChange(issueChangeDto); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.IssueChange issueChange = getSingleMessage(); + assertThat(issueChange.getKey()).isEqualTo(issueChangeDto.getKey()); + assertThat(issueChange.getIssueUuid()).isEqualTo(issueChangeDto.getIssueKey()); + assertThat(issueChange.getChangeData()).isEqualTo(issueChangeDto.getChangeData()); + assertThat(issueChange.getChangeType()).isEqualTo(issueChangeDto.getChangeType()); + assertThat(issueChange.getUserUuid()).isEqualTo(issueChangeDto.getUserUuid()); + assertThat(issueChange.getCreatedAt()).isEqualTo(issueChangeDto.getIssueChangeCreationDate()); + assertThat(issueChange.getProjectUuid()).isEqualTo(issueChangeDto.getProjectUuid()); + } + + @Test + public void execute_exports_issues_by_oldest_create_date_first() { + long createdAt = 1_999_993L; + long now = createdAt + 1_000_000_000L; + String key1 = insertIssueChange(ISSUE_OPEN_UUID, createdAt, now).getKey(); + String key2 = insertIssueChange(ISSUE_OPEN_UUID, createdAt - 500, now + 100).getKey(); + String key3 = insertIssueChange(ISSUE_OPEN_UUID, createdAt - 1000, now + 200).getKey(); + String key4 = insertIssueChange(ISSUE_OPEN_UUID, createdAt + 200, now + 300).getKey(); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES_CHANGELOG)) + .extracting(ProjectDump.IssueChange::getKey) + .containsExactly(key3, key2, key1, key4); + } + + @Test + public void execute_sets_missing_fields_to_default_values() { + long createdAt = 1_999_888L; + insertIssueChange(new IssueChangeDto().setUuid(Uuids.createFast()).setIssueKey(ISSUE_REOPENED_UUID).setCreatedAt(createdAt).setProjectUuid("project_uuid")); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.IssueChange issueChange = getSingleMessage(); + assertThat(issueChange.getKey()).isEmpty(); + assertThat(issueChange.getChangeType()).isEmpty(); + assertThat(issueChange.getChangeData()).isEmpty(); + assertThat(issueChange.getUserUuid()).isEmpty(); + assertThat(issueChange.getCreatedAt()).isEqualTo(createdAt); + } + + @Test + public void execute_sets_createAt_to_zero_if_both_createdAt_and_issueChangeDate_are_null() { + insertIssueChange(new IssueChangeDto().setUuid(Uuids.createFast()).setIssueKey(ISSUE_REOPENED_UUID).setProjectUuid(PROJECT_UUID)); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.IssueChange issueChange = getSingleMessage(); + assertThat(issueChange.getCreatedAt()).isEqualTo(0L); + } + + @Test + public void execute_writes_entries_of_closed_issue() { + insertIssueChange(ISSUE_CLOSED_UUID); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES_CHANGELOG)).isEmpty(); + } + + @Test + public void execute_logs_number_total_exported_issue_changes_count_when_successful() { + insertIssueChange(ISSUE_OPEN_UUID); + insertIssueChange(ISSUE_CONFIRMED_UUID); + insertIssueChange(ISSUE_REOPENED_UUID); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).containsExactly("3 issue changes exported"); + } + + @Test + public void execute_throws_ISE_when_exception_occurs_and_message_contains_number_of_successfully_processed_files_in_() { + insertIssueChange(ISSUE_OPEN_UUID); + insertIssueChange(ISSUE_CONFIRMED_UUID); + insertIssueChange(ISSUE_REOPENED_UUID); + dumpWriter.failIfMoreThan(2, DumpElement.ISSUES_CHANGELOG); + TestComputationStepContext context = new TestComputationStepContext(); + + assertThatThrownBy(() -> underTest.execute(context)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Issues changelog export failed after processing 2 issue changes successfully"); + } + + private void insertIssueChange(String issueUuid) { + insertIssueChange(issueUuid, System2.INSTANCE.now(), null); + } + + private IssueChangeDto insertIssueChange(String issueUuid, long creationDate) { + return insertIssueChange(issueUuid, creationDate, null); + } + + private IssueChangeDto insertIssueChange(String issueUuid, long creationDate, @Nullable Long issueChangeDate) { + IssueChangeDto dto = new IssueChangeDto() + .setKey("uuid_" + issueChangeUuidGenerator++) + .setUuid(Uuids.createFast()) + .setCreatedAt(creationDate) + .setIssueKey(issueUuid) + .setProjectUuid("project_uuid"); + if (issueChangeDate != null) { + dto.setIssueChangeCreationDate(issueChangeDate); + } + return insertIssueChange(dto); + } + + private IssueChangeDto insertIssueChange(IssueChangeDto dto) { + dbClient.issueChangeDao().insert(dbSession, dto); + dbSession.commit(); + return dto; + } + + private void insertIssue(String projectUuid, String uuid, String status) { + IssueDto dto = new IssueDto() + .setKee(uuid) + .setProjectUuid(projectUuid) + .setStatus(status); + dbClient.issueDao().insert(dbSession, dto); + dbSession.commit(); + } + + private ProjectDump.IssueChange getSingleMessage() { + List messages = dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES_CHANGELOG); + assertThat(messages).hasSize(1); + return messages.get(0); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStepTest.java new file mode 100644 index 00000000000..0792c73f139 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStepTest.java @@ -0,0 +1,391 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.issue; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Random; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.sonar.api.issue.Issue; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rules.RuleType; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.projectexport.component.MutableComponentRepository; +import org.sonar.ce.task.projectexport.rule.RuleRepository; +import org.sonar.ce.task.projectexport.rule.RuleRepositoryImpl; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.issue.IssueDto; +import org.sonar.db.project.ProjectDto; +import org.sonar.db.protobuf.DbIssues; +import org.sonar.db.protobuf.DbIssues.Locations; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleDto.Scope; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.api.issue.Issue.STATUS_CONFIRMED; +import static org.sonar.api.issue.Issue.STATUS_OPEN; +import static org.sonar.api.issue.Issue.STATUS_REOPENED; +import static org.sonar.api.issue.Issue.STATUS_RESOLVED; +import static org.sonar.test.ExceptionCauseMatcher.hasType; + +@RunWith(DataProviderRunner.class) +public class ExportIssuesStepTest { + private static final String SOME_PROJECT_UUID = "project uuid"; + private static final String PROJECT_KEY = "projectkey"; + private static final String SOME_REPO = "rule repo"; + private static final String READY_RULE_KEY = "rule key 1"; + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public LogTester logTester = new LogTester(); + + private final DbClient dbClient = dbTester.getDbClient(); + private final DbSession dbSession = dbClient.openSession(false); + private final ProjectHolder projectHolder = mock(ProjectHolder.class); + private final FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private final RuleRepository ruleRepository = new RuleRepositoryImpl(); + private final MutableComponentRepository componentRepository = new ComponentRepositoryImpl(); + + private final ExportIssuesStep underTest = new ExportIssuesStep(dbClient, projectHolder, dumpWriter, ruleRepository, componentRepository); + + private RuleDto readyRuleDto; + + @Before + public void setUp() { + ProjectDto project = createProject(); + when(projectHolder.projectDto()).thenReturn(project); + when(projectHolder.branches()).thenReturn(newArrayList( + new BranchDto().setBranchType(BranchType.BRANCH).setKey("master").setProjectUuid(SOME_PROJECT_UUID).setUuid(SOME_PROJECT_UUID))); + + // adds a random number of Rules to db and repository so that READY_RULE_KEY does always get id=ref=1 + for (int i = 0; i < new Random().nextInt(150); i++) { + RuleKey ruleKey = RuleKey.of("repo_" + i, "key_" + i); + RuleDto ruleDto = insertRule(ruleKey.toString()); + ruleRepository.register(ruleDto.getUuid(), ruleKey); + } + this.readyRuleDto = insertRule(READY_RULE_KEY); + componentRepository.register(12, SOME_PROJECT_UUID, false); + } + + @After + public void tearDown() { + dbSession.close(); + } + + @Test + public void getDescription_is_set() { + assertThat(underTest.getDescription()).isEqualTo("Export issues"); + } + + @Test + public void execute_written_writes_no_issues_when_project_has_no_issues() { + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES)).isEmpty(); + } + + @Test + public void execute_written_writes_no_issues_when_project_has_only_CLOSED_issues() { + insertIssue(readyRuleDto, SOME_PROJECT_UUID, Issue.STATUS_CLOSED); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES)).isEmpty(); + } + + @Test + public void execute_fails_with_ISE_if_componentUuid_is_not_set() { + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID).setComponentUuid(null)); + + expectExportFailure(); + expectedException.expectCause(hasType(NullPointerException.class).andMessage("uuid can not be null")); + + underTest.execute(new TestComputationStepContext()); + } + + @DataProvider + public static Object[][] allStatusesButCLOSED() { + return new Object[][] { + {STATUS_OPEN}, + {STATUS_CONFIRMED}, + {STATUS_REOPENED}, + {STATUS_RESOLVED} + }; + } + + @Test + @UseDataProvider("allStatusesButCLOSED") + public void execute_writes_issues_with_any_status_but_CLOSED(String status) { + String uuid = insertIssue(readyRuleDto, SOME_PROJECT_UUID, status).getKey(); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES)) + .extracting(ProjectDump.Issue::getUuid) + .containsOnly(uuid); + } + + @Test + public void execute_writes_issues_from_any_component_in_project_are_written() { + componentRepository.register(13, "module uuid", false); + componentRepository.register(14, "dir uuid", false); + componentRepository.register(15, "file uuid", false); + String projectIssueUuid = insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID)).getKey(); + String moduleIssueUuid = insertIssue(createBaseIssueDto(readyRuleDto, "module uuid")).getKey(); + String dirIssueUuid = insertIssue(createBaseIssueDto(readyRuleDto, "dir uuid")).getKey(); + String fileIssueUuid = insertIssue(createBaseIssueDto(readyRuleDto, "file uuid")).getKey(); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES)) + .extracting(ProjectDump.Issue::getUuid) + .containsOnly(projectIssueUuid, moduleIssueUuid, dirIssueUuid, fileIssueUuid); + } + + @Test + public void execute_ignores_issues_of_other_projects() { + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID).setProjectUuid("other project")); + String projectIssueUuid = insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID)).getKey(); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES)) + .extracting(ProjectDump.Issue::getUuid) + .containsOnly(projectIssueUuid); + } + + @Test + public void verify_field_by_field_mapping() { + String componentUuid = "component uuid"; + long componentRef = 5454; + componentRepository.register(componentRef, componentUuid, false); + IssueDto issueDto = new IssueDto() + .setKee("issue uuid") + .setComponentUuid(componentUuid) + .setType(988) + .setMessage("msg") + .setLine(10) + .setChecksum("checksum") + .setResolution("resolution") + .setSeverity("severity") + .setManualSeverity(true) + .setGap(13.13d) + .setEffort(99L) + .setAssigneeUuid("assignee-uuid") + .setAuthorLogin("author") + .setTagsString("tags") + .setIssueAttributes("attributes") + .setIssueCreationTime(963L) + .setIssueUpdateTime(852L) + .setIssueCloseTime(741L); + + // fields tested separately and/or required to match SQL request + issueDto + .setType(RuleType.CODE_SMELL) + .setLocations(Locations.newBuilder().addFlow(DbIssues.Flow.newBuilder()).build()) + .setRuleUuid(readyRuleDto.getUuid()) + .setStatus(STATUS_OPEN).setProjectUuid(SOME_PROJECT_UUID); + + insertIssue(issueDto); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Issue issue = getWrittenIssue(); + + assertThat(issue.getUuid()).isEqualTo(issueDto.getKey()); + assertThat(issue.getComponentRef()).isEqualTo(componentRef); + assertThat(issue.getType()).isEqualTo(issueDto.getType()); + assertThat(issue.getMessage()).isEqualTo(issueDto.getMessage()); + assertThat(issue.getLine()).isEqualTo(issueDto.getLine()); + assertThat(issue.getChecksum()).isEqualTo(issueDto.getChecksum()); + assertThat(issue.getStatus()).isEqualTo(issueDto.getStatus()); + assertThat(issue.getResolution()).isEqualTo(issueDto.getResolution()); + assertThat(issue.getSeverity()).isEqualTo(issueDto.getSeverity()); + assertThat(issue.getManualSeverity()).isEqualTo(issueDto.isManualSeverity()); + assertThat(issue.getGap()).isEqualTo(issueDto.getGap()); + assertThat(issue.getEffort()).isEqualTo(issueDto.getEffort()); + assertThat(issue.getAssignee()).isEqualTo(issueDto.getAssigneeUuid()); + assertThat(issue.getAuthor()).isEqualTo(issueDto.getAuthorLogin()); + assertThat(issue.getTags()).isEqualTo(issueDto.getTagsString()); + assertThat(issue.getAttributes()).isEqualTo(issueDto.getIssueAttributes()); + assertThat(issue.getIssueCreatedAt()).isEqualTo(issueDto.getIssueCreationTime()); + assertThat(issue.getIssueUpdatedAt()).isEqualTo(issueDto.getIssueUpdateTime()); + assertThat(issue.getIssueClosedAt()).isEqualTo(issueDto.getIssueCloseTime()); + assertThat(issue.getLocations()).isNotEmpty(); + } + + @Test + public void verify_mapping_of_nullable_numerical_fields_to_defaultValue() { + insertIssue(readyRuleDto, SOME_PROJECT_UUID, STATUS_OPEN); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Issue issue = getWrittenIssue(); + + assertThat(issue.getLine()).isEqualTo(DumpElement.ISSUES.NO_LINE); + assertThat(issue.getGap()).isEqualTo(DumpElement.ISSUES.NO_GAP); + assertThat(issue.getEffort()).isEqualTo(DumpElement.ISSUES.NO_EFFORT); + assertThat(issue.getIssueCreatedAt()).isEqualTo(DumpElement.NO_DATETIME); + assertThat(issue.getIssueUpdatedAt()).isEqualTo(DumpElement.NO_DATETIME); + assertThat(issue.getIssueClosedAt()).isEqualTo(DumpElement.NO_DATETIME); + } + + @Test + public void ruleRef_is_ref_provided_by_RuleRepository() { + + IssueDto issueDto = insertIssue(readyRuleDto, SOME_PROJECT_UUID, STATUS_OPEN); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Issue issue = getWrittenIssue(); + assertThat(issue.getRuleRef()) + .isEqualTo(ruleRepository.register(issueDto.getRuleUuid(), readyRuleDto.getKey()).getRef()); + } + + @Test + public void locations_is_not_set_in_protobuf_if_null_in_DB() { + insertIssue(readyRuleDto, SOME_PROJECT_UUID, STATUS_OPEN); + + underTest.execute(new TestComputationStepContext()); + + assertThat(getWrittenIssue().getLocations()).isEmpty(); + } + + @Test + public void execute_fails_with_ISE_if_locations_cannot_be_parsed_to_protobuf() throws URISyntaxException, IOException { + byte[] rubbishBytes = getRubbishBytes(); + String uuid = insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID).setLocations(rubbishBytes)).getKey(); + + expectExportFailure(); + expectedException.expectCause( + hasType(IllegalStateException.class).andMessage("Fail to read locations from DB for issue " + uuid)); + + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void execute_logs_number_total_exported_issue_count_when_successful() { + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID)); + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID)); + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID)); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).containsExactly("3 issues exported"); + } + + @Test + public void execute_throws_ISE_with_number_of_successful_exports_before_failure() throws URISyntaxException, IOException { + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID)); + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID)); + insertIssue(createBaseIssueDto(readyRuleDto, SOME_PROJECT_UUID).setLocations(getRubbishBytes())).getKey(); + + expectExportFailure(2); + + underTest.execute(new TestComputationStepContext()); + } + + private byte[] getRubbishBytes() throws IOException, URISyntaxException { + return FileUtils.readFileToByteArray(new File(getClass().getResource("rubbish_data.txt").toURI())); + } + + private ProjectDump.Issue getWrittenIssue() { + return dumpWriter.getWrittenMessagesOf(DumpElement.ISSUES).get(0); + } + + private void expectExportFailure() { + expectExportFailure(0); + } + + private void expectExportFailure(int i) { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Issue export failed after processing " + i + " issues successfully"); + } + + private int issueUuidGenerator = 1; + + private IssueDto insertIssue(RuleDto ruleDto, String componentUuid, String status) { + IssueDto dto = createBaseIssueDto(ruleDto, componentUuid, status); + return insertIssue(dto); + } + + private IssueDto insertIssue(IssueDto dto) { + dbClient.issueDao().insert(dbSession, dto); + dbSession.commit(); + return dto; + } + + private ProjectDto createProject() { + ComponentDto projectDto = dbTester.components().insertPrivateProject(c -> c.setDbKey(PROJECT_KEY).setUuid(SOME_PROJECT_UUID)); + dbTester.commit(); + return dbTester.components().getProjectDto(projectDto); + } + + private IssueDto createBaseIssueDto(RuleDto ruleDto, String componentUuid) { + return createBaseIssueDto(ruleDto, componentUuid, STATUS_OPEN); + } + + private IssueDto createBaseIssueDto(RuleDto ruleDto, String componentUuid, String status) { + return new IssueDto() + .setKee("issue_uuid_" + issueUuidGenerator++) + .setComponentUuid(componentUuid) + .setProjectUuid(SOME_PROJECT_UUID) + .setRuleUuid(ruleDto.getUuid()) + .setCreatedAt(System2.INSTANCE.now()) + .setStatus(status); + } + + private RuleDto insertRule(String ruleKey1) { + RuleDto dto = new RuleDto().setRepositoryKey(SOME_REPO).setScope(Scope.MAIN).setRuleKey(ruleKey1).setStatus(RuleStatus.READY); + dbTester.rules().insert(dto.getDefinition()); + dbSession.commit(); + return dto; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStepTest.java new file mode 100644 index 00000000000..06ecfd7d1dc --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportAdHocRulesStepTest.java @@ -0,0 +1,232 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import com.google.common.collect.ImmutableList; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.Date; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.projectexport.steps.ProjectHolder; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.issue.IssueDto; +import org.sonar.db.project.ProjectDto; +import org.sonar.db.rule.RuleDefinitionDto; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleMetadataDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; + +public class ExportAdHocRulesStepTest { + private static final String PROJECT_UUID = "some-uuid"; + + private static final ComponentDto PROJECT = new ComponentDto() + // no id yet + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.PROJECT) + .setDbKey("the_project") + .setName("The Project") + .setDescription("The project description") + .setEnabled(true) + .setUuid(PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid(PROJECT_UUID) + .setModuleUuid(null) + .setModuleUuidPath("." + PROJECT_UUID + ".") + .setProjectUuid(PROJECT_UUID); + + private static final List BRANCHES = ImmutableList.of( + new BranchDto().setBranchType(BranchType.PULL_REQUEST).setProjectUuid(PROJECT_UUID).setKey("pr-1").setUuid("pr-1-uuid").setMergeBranchUuid("master"), + new BranchDto().setBranchType(BranchType.BRANCH).setProjectUuid(PROJECT_UUID).setKey("branch-2").setUuid("branch-2-uuid").setMergeBranchUuid("master") + .setExcludeFromPurge(true), + new BranchDto().setBranchType(BranchType.BRANCH).setProjectUuid(PROJECT_UUID).setKey("branch-3").setUuid("branch-3-uuid").setMergeBranchUuid("master") + .setExcludeFromPurge(false)); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public LogTester logTester = new LogTester(); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private int issueUuidGenerator = 1; + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private ProjectHolder projectHolder = mock(ProjectHolder.class); + private ExportAdHocRulesStep underTest = new ExportAdHocRulesStep(dbTester.getDbClient(), projectHolder, dumpWriter); + + @Before + public void setup() { + ProjectDto project = createProject(); + when(projectHolder.projectDto()).thenReturn(project); + } + + @Test + public void export_zero_ad_hoc_rules() { + underTest.execute(new TestComputationStepContext()); + + List exportedRules = dumpWriter.getWrittenMessagesOf(DumpElement.AD_HOC_RULES); + assertThat(exportedRules).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 ad-hoc rules exported"); + } + + @Test + public void execute_only_exports_ad_hoc_rules_that_reference_project_issue() { + String differentProject = "diff-proj-uuid"; + RuleDto rule1 = insertRule(RuleKey.of("plugin-1", "rule-1"), true, true); + RuleDto rule2 = insertRule(RuleKey.of("plugin-1", "rule-2"), true, true); + insertRule(RuleKey.of("plugin-1", "rule-3"), true, true); + insertIssue(rule1, differentProject, differentProject); + insertIssue(rule2, PROJECT_UUID, PROJECT_UUID); + + underTest.execute(new TestComputationStepContext()); + + List exportedRules = dumpWriter.getWrittenMessagesOf(DumpElement.AD_HOC_RULES); + assertThat(exportedRules).hasSize(1); + assertThat(exportedRules).extracting(ProjectDump.AdHocRule::getPluginName, ProjectDump.AdHocRule::getPluginRuleKey) + .containsOnly(tuple("plugin-1", "rule-2")); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("1 ad-hoc rules exported"); + } + + @Test + public void execute_only_exports_rules_that_are_ad_hoc() { + RuleDto rule1 = insertRule(RuleKey.of("plugin-1", "rule-1"), false, false); + RuleDto rule2 = insertRule(RuleKey.of("plugin-1", "rule-2"), true, false); + RuleDto rule3 = insertRule(RuleKey.of("plugin-1", "rule-3"), true, true); + insertIssue(rule1, PROJECT_UUID, PROJECT_UUID); + insertIssue(rule2, PROJECT_UUID, PROJECT_UUID); + insertIssue(rule3, PROJECT_UUID, PROJECT_UUID); + + underTest.execute(new TestComputationStepContext()); + + List exportedRules = dumpWriter.getWrittenMessagesOf(DumpElement.AD_HOC_RULES); + assertThat(exportedRules).hasSize(1); + assertThat(exportedRules).extracting(ProjectDump.AdHocRule::getPluginName, ProjectDump.AdHocRule::getPluginRuleKey) + .containsOnly(tuple("plugin-1", "rule-3")); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("1 ad-hoc rules exported"); + } + + @Test + public void execute_exports_ad_hoc_rules_that_are_referenced_by_issues_on_branches_excluded_from_purge() { + when(projectHolder.branches()).thenReturn(BRANCHES); + RuleDto rule1 = insertRule(RuleKey.of("plugin-1", "rule-1"), true, true); + RuleDto rule2 = insertRule(RuleKey.of("plugin-1", "rule-2"), true, true); + RuleDto rule3 = insertRule(RuleKey.of("plugin-1", "rule-3"), true, true); + insertIssue(rule1, "branch-1-uuid", "branch-1-uuid"); + insertIssue(rule2, "branch-2-uuid", "branch-2-uuid"); + insertIssue(rule3, "branch-3-uuid", "branch-3-uuid"); + + underTest.execute(new TestComputationStepContext()); + + List exportedRules = dumpWriter.getWrittenMessagesOf(DumpElement.AD_HOC_RULES); + assertThat(exportedRules).hasSize(1); + assertThat(exportedRules).extracting(ProjectDump.AdHocRule::getPluginName, ProjectDump.AdHocRule::getPluginRuleKey) + .containsOnly(tuple("plugin-1", "rule-2")); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("1 ad-hoc rules exported"); + } + + @Test + public void execute_throws_ISE_with_number_of_successful_exports_before_failure() { + RuleDto rule1 = insertRule(RuleKey.of("plugin-1", "rule-1"), true, true); + RuleDto rule2 = insertRule(RuleKey.of("plugin-1", "rule-2"), true, true); + RuleDto rule3 = insertRule(RuleKey.of("plugin-1", "rule-3"), true, true); + insertIssue(rule1, PROJECT_UUID, PROJECT_UUID); + insertIssue(rule2, PROJECT_UUID, PROJECT_UUID); + insertIssue(rule3, PROJECT_UUID, PROJECT_UUID); + dumpWriter.failIfMoreThan(2, DumpElement.AD_HOC_RULES); + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Ad-hoc rules export failed after processing 2 rules successfully"); + + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Export ad-hoc rules"); + } + + private ProjectDto createProject() { + Date createdAt = new Date(); + ComponentDto projectDto = dbTester.components().insertPublicProject(PROJECT); + BRANCHES.forEach(branch -> dbTester.components().insertProjectBranch(projectDto, branch).setCreatedAt(createdAt)); + dbTester.commit(); + return dbTester.components().getProjectDto(projectDto); + } + + private void insertIssue(RuleDto ruleDto, String projectUuid, String componentUuid) { + IssueDto dto = createBaseIssueDto(ruleDto, projectUuid, componentUuid); + insertIssue(dto); + } + + private void insertIssue(IssueDto dto) { + dbTester.getDbClient().issueDao().insert(dbTester.getSession(), dto); + dbTester.commit(); + } + + private IssueDto createBaseIssueDto(RuleDto ruleDto, String projectUuid, String componentUuid) { + return new IssueDto() + .setKee("issue_uuid_" + issueUuidGenerator++) + .setComponentUuid(componentUuid) + .setProjectUuid(projectUuid) + .setRuleUuid(ruleDto.getUuid()) + .setStatus("OPEN"); + } + + private RuleDto insertRule(RuleKey ruleKey, boolean isExternal, boolean isAdHoc) { + dbTester.rules().insert( + new RuleDefinitionDto() + .setRuleKey(ruleKey) + .setIsExternal(isExternal) + .setIsAdHoc(isAdHoc) + .setStatus(RuleStatus.READY) + .setScope(RuleDto.Scope.ALL)); + dbTester.commit(); + return dbTester.getDbClient().ruleDao().selectByKey(dbTester.getSession(), ruleKey) + .orElseThrow(() -> new RuntimeException("insertAdHocRule failed")); + } + + private void insertRuleMetadata(String ruleUuid, String adHocName) { + dbTester.rules().insertOrUpdateMetadata(new RuleMetadataDto() + .setRuleUuid(ruleUuid) + .setAdHocName(adHocName)); + dbTester.commit(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportRuleStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportRuleStepTest.java new file mode 100644 index 00000000000..d2983b7e765 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/ExportRuleStepTest.java @@ -0,0 +1,123 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.steps.DumpElement; +import org.sonar.ce.task.projectexport.steps.FakeDumpWriter; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.core.util.Uuids; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ExportRuleStepTest { + private static final String REPOSITORY = "repository"; + + @org.junit.Rule + public LogTester logTester = new LogTester(); + @org.junit.Rule + public ExpectedException expectedException = ExpectedException.none(); + + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private SimpleRuleRepository ruleRepository = new SimpleRuleRepository(); + private ExportRuleStep underTest = new ExportRuleStep(ruleRepository, dumpWriter); + + @Test + public void getDescription_is_set() { + assertThat(underTest.getDescription()).isEqualTo("Export rules"); + } + + @Test + public void execute_writes_no_rules_when_repository_is_empty() { + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.RULES)).isEmpty(); + } + + @Test + public void execute_writes_all_rules_in_order_returned_by_repository() { + String[] keys = new String[10]; + for (int i = 0; i < 10; i++) { + String key = "key_" + i; + ruleRepository.add(key); + keys[i] = key; + } + + underTest.execute(new TestComputationStepContext()); + + List rules = dumpWriter.getWrittenMessagesOf(DumpElement.RULES); + assertThat(rules).extracting(ProjectDump.Rule::getKey).containsExactly(keys); + assertThat(rules).extracting(ProjectDump.Rule::getRepository).containsOnly(REPOSITORY); + } + + @Test + public void execute_logs_number_total_exported_rules_count_when_successful() { + ruleRepository.add("A").add("B").add("C").add("D"); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).containsExactly("4 rules exported"); + } + + @Test + public void excuse_throws_ISE_exception_with_number_of_successfully_exported_rules() { + ruleRepository.add("A").add("B").add("C") + // will cause NPE + .addNull(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Rule Export failed after processing 3 rules successfully"); + + underTest.execute(new TestComputationStepContext()); + } + + private static class SimpleRuleRepository implements RuleRepository { + + private List rules = new ArrayList<>(); + + @Override + public Rule register(String uuid, RuleKey ruleKey) { + throw new UnsupportedOperationException("getByRuleKey not implemented"); + } + + public SimpleRuleRepository add(String key) { + this.rules.add(new Rule(Uuids.createFast(), REPOSITORY, key)); + return this; + } + + public SimpleRuleRepository addNull() { + this.rules.add(null); + return this; + } + + @Override + public Collection getAll() { + return rules; + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImplTest.java new file mode 100644 index 00000000000..cc94dab3987 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleRepositoryImplTest.java @@ -0,0 +1,139 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import java.util.Collection; +import java.util.Random; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.rule.RuleKey; +import org.sonar.core.util.Uuids; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RuleRepositoryImplTest { + private static final String SOME_UUID = "uuid-846"; + private static final String SOME_REPOSITORY = "rep"; + private static final String SOME_RULE_KEY = "key"; + private static final Rule SOME_RULE = new Rule("uuid-1", SOME_REPOSITORY, SOME_RULE_KEY); + + @org.junit.Rule + public ExpectedException expectedException = ExpectedException.none(); + + private Random random = new Random(); + + private RuleRepositoryImpl underTest = new RuleRepositoryImpl(); + + @Test + public void register_throws_NPE_if_ruleKey_is_null() { + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("ruleKey can not be null"); + + underTest.register(SOME_UUID, null); + } + + @Test + public void register_does_not_enforce_some_RuleKey_is_registered_under_a_single_id() { + underTest.register(SOME_UUID, RuleKey.of(SOME_REPOSITORY, SOME_RULE_KEY)); + for (int i = 0; i < someRandomInt(); i++) { + Rule otherRule = underTest.register(Integer.toString(i), RuleKey.of(SOME_REPOSITORY, SOME_RULE_KEY)); + assertThat(otherRule.getRef()).isEqualTo(Integer.toString(i)); + assertThat(otherRule.getRepository()).isEqualTo(SOME_REPOSITORY); + assertThat(otherRule.getKey()).isEqualTo(SOME_RULE_KEY); + } + } + + @Test + public void register_fails_IAE_if_RuleKey_is_not_the_same_repository_for_a_specific_ref() { + underTest.register(SOME_UUID, RuleKey.of(SOME_REPOSITORY, SOME_RULE_KEY)); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Specified RuleKey 'other repo:key' is not equal to the one already registered in repository for ref " + SOME_UUID + ": 'rep:key'"); + + underTest.register(SOME_UUID, RuleKey.of("other repo", SOME_RULE_KEY)); + } + + @Test + public void register_fails_IAE_if_RuleKey_is_not_the_same_key_for_a_specific_ref() { + underTest.register(SOME_UUID, RuleKey.of(SOME_REPOSITORY, SOME_RULE_KEY)); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Specified RuleKey 'rep:other key' is not equal to the one already registered in repository for ref " + SOME_UUID + ": 'rep:key'"); + + underTest.register(SOME_UUID, RuleKey.of(SOME_REPOSITORY, "other key")); + } + + @Test + public void register_returns_the_same_object_for_every_call_with_equals_RuleKey_objects() { + Rule rule = underTest.register(SOME_UUID, RuleKey.of(SOME_REPOSITORY, SOME_RULE_KEY)); + for (int i = 0; i < someRandomInt(); i++) { + assertThat(underTest.register(Uuids.createFast(), RuleKey.of(SOME_REPOSITORY, SOME_RULE_KEY)).getRef()).isNotEqualTo(rule.getRef()); + } + } + + @Test + public void register_returns_Rule_object_created_from_arguments() { + for (int i = 0; i < someRandomInt(); i++) { + String repository = SOME_REPOSITORY + i; + String ruleKey = String.valueOf(i); + Rule rule = underTest.register(Integer.toString(i), RuleKey.of(repository, ruleKey)); + assertThat(rule.getRef()).isEqualTo(Integer.toString(i)); + assertThat(rule.getRepository()).isEqualTo(repository); + assertThat(rule.getKey()).isEqualTo(ruleKey); + } + } + + @Test + public void getAll_returns_immutable_empty_collection_when_register_was_never_called() { + Collection all = underTest.getAll(); + assertThat(all).isEmpty(); + + ensureImmutable(all); + } + + @Test + public void getAll_returns_immutable_collection_with_one_Rule_for_each_distinct_RuleKey() { + int size = someRandomInt(); + String[] repositories = new String[size]; + String[] keys = new String[size]; + for (int i = 0; i < size; i++) { + String key = "key_" + i; + String repository = "repo_" + i; + underTest.register(Uuids.createFast(), RuleKey.of(repository, key)); + repositories[i] = repository; + keys[i] = key; + } + + Collection all = underTest.getAll(); + + assertThat(all).extracting(Rule::getRepository).containsOnly(repositories); + assertThat(all).extracting(Rule::getKey).containsOnly(keys); + ensureImmutable(all); + } + + private void ensureImmutable(Collection collection) { + expectedException.expect(UnsupportedOperationException.class); + collection.add(SOME_RULE); + } + + private int someRandomInt() { + return 50 + Math.abs(random.nextInt(500)); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleTest.java new file mode 100644 index 00000000000..f33cff5dcc6 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/rule/RuleTest.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.rule; + +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RuleTest { + private static final String SOME_DUMP_UUID = "uuid-12334"; + private static final String SOME_REPOSITORY = "some repository"; + private static final String SOME_KEY = "some key"; + + @org.junit.Rule + public ExpectedException expectedException = ExpectedException.none(); + + private Rule underTest = new Rule(SOME_DUMP_UUID, SOME_REPOSITORY, SOME_KEY); + + @Test + public void constructor_throws_NPE_if_repository_is_null() { + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("repository can not be null"); + + new Rule(SOME_DUMP_UUID, null, SOME_KEY); + } + + @Test + public void constructor_throws_NPE_if_key_is_null() { + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("key can not be null"); + + new Rule(SOME_DUMP_UUID, SOME_REPOSITORY, null); + } + + @Test + public void equals_compares_repository_and_key() { + assertThat(underTest).isNotEqualTo(new Rule(SOME_DUMP_UUID, SOME_KEY, SOME_REPOSITORY)); + assertThat(underTest).isNotEqualTo(new Rule(SOME_DUMP_UUID, "other repository", SOME_KEY)); + assertThat(underTest).isNotEqualTo(new Rule(SOME_DUMP_UUID, SOME_REPOSITORY, "other key")); + } + + @Test + public void equals_ignores_dump_id() { + assertThat(underTest).isEqualTo(new Rule("uuid-8888", SOME_REPOSITORY, SOME_KEY)); + } + + @Test + public void hashcode_is_based_on_repository_and_key() { + assertThat(underTest.hashCode()).isEqualTo(new Rule(SOME_DUMP_UUID, SOME_REPOSITORY, SOME_KEY).hashCode()); + assertThat(underTest.hashCode()).isNotEqualTo(new Rule(SOME_DUMP_UUID, SOME_KEY, SOME_REPOSITORY).hashCode()); + assertThat(underTest.hashCode()).isNotEqualTo(new Rule(SOME_DUMP_UUID, "other repository", SOME_KEY).hashCode()); + assertThat(underTest.hashCode()).isNotEqualTo(new Rule(SOME_DUMP_UUID, SOME_REPOSITORY, "other key").hashCode()); + } + + @Test + public void hashcode_ignores_dump_id() { + assertThat(underTest.hashCode()).isEqualTo(new Rule("uuid-8888", SOME_REPOSITORY, SOME_KEY).hashCode()); + } + + @Test + public void toString_displays_all_fields() { + assertThat(underTest.toString()).isEqualTo("Rule{ref='uuid-12334', repository='some repository', key='some key'}"); + + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpElementTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpElementTest.java new file mode 100644 index 00000000000..26ea45f2759 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpElementTest.java @@ -0,0 +1,50 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.ce.task.projectexport.steps.DumpElement.*; + +public class DumpElementTest { + + @Test + public void test_filename() { + assertThat(METADATA.filename()).isEqualTo("metadata.pb"); + assertThat(COMPONENTS.filename()).isEqualTo("components.pb"); + assertThat(MEASURES.filename()).isEqualTo("measures.pb"); + assertThat(METRICS.filename()).isEqualTo("metrics.pb"); + assertThat(ISSUES.filename()).isEqualTo("issues.pb"); + assertThat(ISSUES_CHANGELOG.filename()).isEqualTo("issues_changelog.pb"); + assertThat(RULES.filename()).isEqualTo("rules.pb"); + assertThat(ANALYSES.filename()).isEqualTo("analyses.pb"); + assertThat(SETTINGS.filename()).isEqualTo("settings.pb"); + assertThat(LINKS.filename()).isEqualTo("links.pb"); + assertThat(EVENTS.filename()).isEqualTo("events.pb"); + assertThat(PLUGINS.filename()).isEqualTo("plugins.pb"); + } + + @Test + public void test_parser() { + assertThat(METADATA.parser()).isSameAs(ProjectDump.Metadata.parser()); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpWriterImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpWriterImplTest.java new file mode 100644 index 00000000000..1cd1666bc1c --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/DumpWriterImplTest.java @@ -0,0 +1,152 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.common.collect.Iterables; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.io.File; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.utils.TempFolder; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.projectexport.util.ProjectExportDumpFS; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.ce.task.projectexport.steps.DumpElement.COMPONENTS; + +public class DumpWriterImplTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public TemporaryFolder junitTemp = new TemporaryFolder(); + + @Rule + public LogTester logTester = new LogTester(); + + MutableDumpReaderImpl dumpReader = new MutableDumpReaderImpl(); + ProjectExportDumpFS projectexportDumpFS = mock(ProjectExportDumpFS.class); + TempFolder temp = mock(TempFolder.class); + ProjectDescriptor descriptor = mock(ProjectDescriptor.class); + File rootDir; + File zipFile; + File targetZipFile; + DumpWriter underTest; + + @Before + public void setUp() throws Exception { + rootDir = junitTemp.newFolder(); + dumpReader.setTempRootDir(rootDir); + zipFile = junitTemp.newFile(); + targetZipFile = junitTemp.newFile(); + when(temp.newDir()).thenReturn(rootDir); + when(temp.newFile()).thenReturn(zipFile); + when(projectexportDumpFS.exportDumpOf(descriptor)).thenReturn(targetZipFile); + underTest = new DumpWriterImpl(descriptor, projectexportDumpFS, temp); + } + + @Test + public void writeMetadata_writes_to_file() { + underTest.write(newMetadata()); + + assertThat(dumpReader.metadata().getProjectKey()).isEqualTo("foo"); + } + + @Test + public void writeMetadata_fails_if_called_twice() { + underTest.write(newMetadata()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Metadata has already been written"); + underTest.write(newMetadata()); + } + + @Test + public void publish_zips_directory_then_deletes_the_temp_directory() { + underTest.write(newMetadata()); + underTest.publish(); + + assertThat(rootDir).doesNotExist(); + assertThat(targetZipFile).isFile().exists(); + assertThat(logTester.logs(LoggerLevel.INFO).get(0)) + .contains("Dump file published", "size=", "path=" + targetZipFile.getAbsolutePath()); + } + + @Test + public void publish_fails_if_called_twice() { + underTest.write(newMetadata()); + underTest.publish(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Dump is already published"); + underTest.publish(); + } + + @Test + public void publish_fails_if_metadata_is_missing() { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Metadata is missing"); + underTest.publish(); + } + + @Test + public void ensure_written_fields_can_be_read() { + try (StreamWriter writer = underTest.newStreamWriter(COMPONENTS)) { + writer.write(ProjectDump.Component.newBuilder() + .setKey("abc") + .setScope("FIL") + .setQualifier("FIL") + .setLanguage("java") + .build()); + } + try (MessageStream reader = dumpReader.stream(COMPONENTS)) { + ProjectDump.Component component = Iterables.getOnlyElement(reader); + assertThat(component.getKey()).isEqualTo("abc"); + assertThat(component.getScope()).isEqualTo("FIL"); + assertThat(component.getQualifier()).isEqualTo("FIL"); + assertThat(component.getLanguage()).isEqualTo("java"); + } + } + + @Test + public void create_empty_file_if_stream_is_empty() { + try (StreamWriter writer = underTest.newStreamWriter(COMPONENTS)) { + // no components + } + assertThat(new File(rootDir, COMPONENTS.filename())).isFile().exists(); + try (MessageStream reader = dumpReader.stream(COMPONENTS)) { + assertThat(reader).isEmpty(); + } + } + + private static ProjectDump.Metadata newMetadata() { + return ProjectDump.Metadata.newBuilder() + .setProjectKey("foo") + .build(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportEventsStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportEventsStepTest.java new file mode 100644 index 00000000000..bfac265a168 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportEventsStepTest.java @@ -0,0 +1,170 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.SnapshotDto; +import org.sonar.db.event.EventDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; +import static org.sonar.db.component.SnapshotDto.STATUS_PROCESSED; + +public class ExportEventsStepTest { + + private static final long NOW = 1_450_000_000_000L; + private static final long IN_THE_PAST = 1_440_000_000_000L; + + private static final String PROJECT_UUID = "project_uuid"; + private static final ComponentDto PROJECT = new ComponentDto() + .setUuid(PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid(PROJECT_UUID) + .setProjectUuid(PROJECT_UUID) + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.PROJECT) + .setDbKey("the_project") + .setEnabled(true); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + @Rule + public LogTester logTester = new LogTester(); + + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private MutableProjectHolder projectHolder = new MutableProjectHolderImpl(); + private ComponentRepositoryImpl componentRepository = new ComponentRepositoryImpl(); + private ExportEventsStep underTest = new ExportEventsStep(dbTester.getDbClient(), projectHolder, componentRepository, dumpWriter); + + @Before + public void setUp() { + ComponentDto projectDto = dbTester.components().insertPublicProject(PROJECT); + componentRepository.register(1, projectDto.uuid(), false); + projectHolder.setProjectDto(dbTester.components().getProjectDto(projectDto)); + } + + @Test + public void export_zero_events() { + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 events exported"); + List events = dumpWriter.getWrittenMessagesOf(DumpElement.EVENTS); + assertThat(events).isEmpty(); + } + + @Test + public void export_events() { + SnapshotDto snapshot = insertSnapshot(); + insertEvent(snapshot, "E1", "one"); + insertEvent(snapshot, "E2", "two"); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("2 events exported"); + List events = dumpWriter.getWrittenMessagesOf(DumpElement.EVENTS); + assertThat(events).hasSize(2); + assertThat(events).extracting(ProjectDump.Event::getUuid).containsOnly("E1", "E2"); + } + + @Test + public void throws_ISE_if_error() { + SnapshotDto snapshot = insertSnapshot(); + insertEvent(snapshot, "E1", "one"); + insertEvent(snapshot, "E2", "two"); + dumpWriter.failIfMoreThan(1, DumpElement.EVENTS); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Event Export failed after processing 1 events successfully"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void export_all_fields() { + SnapshotDto snapshot = insertSnapshot(); + dbTester.getDbClient().eventDao().insert(dbTester.getSession(), new EventDto() + .setUuid("E1") + .setAnalysisUuid(snapshot.getUuid()) + .setComponentUuid(snapshot.getComponentUuid()) + .setDate(IN_THE_PAST) + .setCreatedAt(NOW) + .setData("data") + .setName("name") + .setCategory("categ") + .setDescription("desc")); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Event event = dumpWriter.getWrittenMessagesOf(DumpElement.EVENTS).get(0); + assertThat(event.getUuid()).isEqualTo("E1"); + assertThat(event.getName()).isEqualTo("name"); + assertThat(event.getData()).isEqualTo("data"); + assertThat(event.getCategory()).isEqualTo("categ"); + assertThat(event.getDescription()).isEqualTo("desc"); + assertThat(event.getDate()).isEqualTo(IN_THE_PAST); + assertThat(event.getAnalysisUuid()).isEqualTo(snapshot.getUuid()); + assertThat(event.getComponentRef()).isEqualTo(1); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isEqualTo("Export events"); + } + + private void insertEvent(SnapshotDto snapshot, String uuid, String name) { + dbTester.getDbClient().eventDao().insert(dbTester.getSession(), new EventDto() + .setUuid(uuid) + .setAnalysisUuid(snapshot.getUuid()) + .setComponentUuid(snapshot.getComponentUuid()) + .setDate(IN_THE_PAST) + .setCreatedAt(NOW) + .setName(name)); + dbTester.commit(); + } + + private SnapshotDto insertSnapshot() { + SnapshotDto snapshot = new SnapshotDto() + .setUuid("U1") + .setComponentUuid(PROJECT.uuid()) + .setStatus(STATUS_PROCESSED) + .setLast(false); + dbTester.getDbClient().snapshotDao().insert(dbTester.getSession(), snapshot); + dbTester.commit(); + return snapshot; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLinksStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLinksStepTest.java new file mode 100644 index 00000000000..86c2baafde0 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLinksStepTest.java @@ -0,0 +1,140 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump.Link; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepository; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ProjectLinkDto; +import org.sonar.db.project.ProjectExportMapper; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.groups.Tuple.tuple; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; + +public class ExportLinksStepTest { + + private static final String PROJECT_UUID = "project_uuid"; + private static final ComponentDto PROJECT = new ComponentDto() + // no id yet + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.PROJECT) + .setDbKey("the_project") + .setName("The Project") + .setDescription("The project description") + .setEnabled(true) + .setUuid(PROJECT_UUID) + .setRootUuid(PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setModuleUuid(null) + .setModuleUuidPath("." + PROJECT_UUID + ".") + .setProjectUuid(PROJECT_UUID); + + @Rule + public DbTester db = DbTester.createWithExtensionMappers(System2.INSTANCE, ProjectExportMapper.class); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public LogTester logTester = new LogTester(); + + private final FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private final ComponentRepository componentRepository = mock(ComponentRepository.class); + private final ProjectHolder projectHolder = mock(ProjectHolder.class); + private final ExportLinksStep underTest = new ExportLinksStep(db.getDbClient(), componentRepository, projectHolder, dumpWriter); + + @Before + public void setUp() { + ComponentDto project = db.components().insertPublicProject(PROJECT); + when(projectHolder.projectDto()).thenReturn(db.components().getProjectDto(project)); + when(componentRepository.getRef(PROJECT_UUID)).thenReturn(1L); + } + + @Test + public void export_zero_links() { + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 links exported"); + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.LINKS)).isEmpty(); + } + + @Test + public void export_links() { + ProjectLinkDto link1 = db.componentLinks().insertCustomLink(PROJECT); + ProjectLinkDto link2 = db.componentLinks().insertProvidedLink(PROJECT); + db.componentLinks().insertCustomLink(db.components().insertPrivateProject()); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("2 links exported"); + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.LINKS)) + .extracting(Link::getUuid, Link::getName, Link::getType, Link::getHref) + .containsExactlyInAnyOrder( + tuple(link1.getUuid(), link1.getName(), link1.getType(), link1.getHref()), + tuple(link2.getUuid(), "", link2.getType(), link2.getHref())); + } + + @Test + public void throws_ISE_if_error() { + db.componentLinks().insertCustomLink(PROJECT); + db.componentLinks().insertProvidedLink(PROJECT); + db.componentLinks().insertProvidedLink(PROJECT); + db.componentLinks().insertCustomLink(db.components().insertPrivateProject()); + + dumpWriter.failIfMoreThan(2, DumpElement.LINKS); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Link export failed after processing 2 link(s) successfully"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void test_all_fields() { + ProjectLinkDto link = db.componentLinks().insertCustomLink(PROJECT, l -> l.setName("name").setHref("href").setType("type")); + + underTest.execute(new TestComputationStepContext()); + + Link reloaded = dumpWriter.getWrittenMessagesOf(DumpElement.LINKS).get(0); + assertThat(reloaded.getUuid()).isEqualTo(link.getUuid()); + assertThat(reloaded.getName()).isEqualTo(link.getName()); + assertThat(reloaded.getHref()).isEqualTo(link.getHref()); + assertThat(reloaded.getType()).isEqualTo(link.getType()); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isEqualTo("Export links"); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStepTest.java new file mode 100644 index 00000000000..b46e772ee77 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportLiveMeasuresStepTest.java @@ -0,0 +1,205 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.metric.MetricDto; +import org.sonar.db.project.ProjectDto; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.api.measures.Metric.ValueType.INT; + +public class ExportLiveMeasuresStepTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public LogTester logTester = new LogTester(); + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private ComponentRepositoryImpl componentRepository = new ComponentRepositoryImpl(); + private MutableMetricRepository metricRepository = new MutableMetricRepositoryImpl(); + private ProjectHolder projectHolder = mock(ProjectHolder.class); + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private ExportLiveMeasuresStep underTest = new ExportLiveMeasuresStep(dbTester.getDbClient(), projectHolder, componentRepository, metricRepository, dumpWriter); + + @Test + public void export_zero_measures() { + when(projectHolder.branches()).thenReturn(newArrayList()); + when(projectHolder.projectDto()).thenReturn(new ProjectDto().setUuid("does not exist")); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.LIVE_MEASURES)).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 live measures exported"); + assertThat(metricRepository.getRefByUuid()).isEmpty(); + } + + @Test + public void export_measures() { + ComponentDto project = createProject(true); + componentRepository.register(1, project.uuid(), false); + MetricDto metric = dbTester.measures().insertMetric(m -> m.setValueType(INT.name())); + dbTester.measures().insertLiveMeasure(project, metric, m -> m.setValue(4711.0d).setVariation(null)); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(project)); + when(projectHolder.branches()).thenReturn(newArrayList(new BranchDto() + .setProjectUuid(project.uuid()) + .setUuid(project.uuid()) + .setKey("master") + .setBranchType(BranchType.BRANCH))); + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.LIVE_MEASURES); + assertThat(exportedMeasures).hasSize(1); + assertThat(exportedMeasures) + .extracting(ProjectDump.LiveMeasure::getMetricRef, m -> m.getDoubleValue().getValue(), ProjectDump.LiveMeasure::hasVariation) + .containsOnly(tuple(0, 4711.0d, false)); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("1 live measures exported"); + assertThat(metricRepository.getRefByUuid().keySet()).containsOnly(metric.getUuid()); + } + + @Test + public void do_not_export_measures_on_disabled_projects() { + ComponentDto project = createProject(false); + componentRepository.register(1, project.uuid(), false); + MetricDto metric = dbTester.measures().insertMetric(m -> m.setValueType(INT.name())); + dbTester.measures().insertLiveMeasure(project, metric, m -> m.setValue(4711.0d)); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(project)); + when(projectHolder.branches()).thenReturn(newArrayList(new BranchDto() + .setProjectUuid(project.uuid()) + .setUuid(project.uuid()) + .setKey("master") + .setBranchType(BranchType.BRANCH))); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.LIVE_MEASURES); + assertThat(exportedMeasures).isEmpty(); + } + + @Test + public void do_not_export_measures_on_disabled_metrics() { + ComponentDto project = createProject(true); + componentRepository.register(1, project.uuid(), false); + MetricDto metric = dbTester.measures().insertMetric(m -> m.setValueType(INT.name()).setEnabled(false)); + dbTester.measures().insertLiveMeasure(project, metric, m -> m.setValue(4711.0d)); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(project)); + when(projectHolder.branches()).thenReturn(newArrayList(new BranchDto() + .setProjectUuid(project.uuid()) + .setUuid(project.uuid()) + .setKey("master") + .setBranchType(BranchType.BRANCH))); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.LIVE_MEASURES); + assertThat(exportedMeasures).isEmpty(); + } + + @Test + public void test_exported_fields() { + ComponentDto project = createProject(true); + componentRepository.register(1, project.uuid(), false); + MetricDto metric = dbTester.measures().insertMetric(m -> m.setValueType(INT.name())); + dbTester.measures().insertLiveMeasure(project, metric, m -> m.setProjectUuid(project.uuid()).setValue(4711.0d).setData("test").setVariation(7.0d)); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(project)); + when(projectHolder.branches()).thenReturn(newArrayList(new BranchDto() + .setProjectUuid(project.uuid()) + .setUuid(project.uuid()) + .setKey("master") + .setBranchType(BranchType.BRANCH))); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.LIVE_MEASURES); + assertThat(exportedMeasures).hasSize(1); + assertThat(exportedMeasures) + .extracting( + ProjectDump.LiveMeasure::getComponentRef, + ProjectDump.LiveMeasure::getMetricRef, + m -> m.getDoubleValue().getValue(), + ProjectDump.LiveMeasure::getTextValue, + m -> m.getVariation().getValue()) + .containsOnly(tuple( + 1L, + 0, + 4711.0d, + "test", + 7.0d)); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("1 live measures exported"); + assertThat(metricRepository.getRefByUuid().keySet()).containsOnly(metric.getUuid()); + } + + @Test + public void test_null_exported_fields() { + ComponentDto project = createProject(true); + componentRepository.register(1, project.uuid(), false); + MetricDto metric = dbTester.measures().insertMetric(m -> m.setValueType(INT.name())); + dbTester.measures().insertLiveMeasure(project, metric, m -> m.setProjectUuid(project.uuid()).setValue(null).setData((String) null).setVariation(null)); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(project)); + when(projectHolder.branches()).thenReturn(newArrayList(new BranchDto() + .setProjectUuid(project.uuid()) + .setUuid(project.uuid()) + .setKey("master") + .setBranchType(BranchType.BRANCH))); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.LIVE_MEASURES); + assertThat(exportedMeasures).hasSize(1); + assertThat(exportedMeasures) + .extracting( + ProjectDump.LiveMeasure::hasDoubleValue, + ProjectDump.LiveMeasure::getTextValue, + ProjectDump.LiveMeasure::hasVariation) + .containsOnly(tuple( + false, + "", + false)); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("1 live measures exported"); + assertThat(metricRepository.getRefByUuid().keySet()).containsOnly(metric.getUuid()); + } + + @Test + public void test_getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Export live measures"); + } + + private ComponentDto createProject(boolean enabled) { + return dbTester.components().insertPrivateProject(p -> p.setEnabled(enabled)); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStepTest.java new file mode 100644 index 00000000000..698d1a5be02 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMeasuresStepTest.java @@ -0,0 +1,235 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.SnapshotDto; +import org.sonar.db.measure.MeasureDto; +import org.sonar.db.metric.MetricDto; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; +import static org.sonar.db.component.ComponentDto.UUID_PATH_SEPARATOR; +import static org.sonar.db.component.SnapshotDto.STATUS_PROCESSED; +import static org.sonar.db.component.SnapshotDto.STATUS_UNPROCESSED; + +public class ExportMeasuresStepTest { + + private static final ComponentDto PROJECT = new ComponentDto() + .setDbKey("project_key") + .setUuid("project_uuid") + .setRootUuid("project_uuid") + .setProjectUuid("project_uuid") + .setUuidPath(UUID_PATH_OF_ROOT) + .setEnabled(true); + private static final ComponentDto FILE = new ComponentDto() + .setDbKey("file_key") + .setUuid("file_uuid") + .setRootUuid("project_uuid") + .setProjectUuid("project_uuid") + .setUuidPath(UUID_PATH_OF_ROOT + PROJECT.uuid() + UUID_PATH_SEPARATOR) + .setEnabled(true); + private static final ComponentDto ANOTHER_PROJECT = new ComponentDto() + .setDbKey("another_project_key") + .setUuid("another_project_uuid") + .setRootUuid("another_project_uuid") + .setProjectUuid("another_project_uuid") + .setUuidPath(UUID_PATH_OF_ROOT) + .setEnabled(true); + + private static final MetricDto NCLOC = new MetricDto() + .setUuid("3") + .setKey("ncloc") + .setShortName("Lines of code") + .setEnabled(true); + + private static final MetricDto DISABLED_METRIC = new MetricDto() + .setUuid("4") + .setKey("coverage") + .setShortName("Coverage") + .setEnabled(false); + + private static final List BRANCHES = newArrayList( + new BranchDto() + .setBranchType(BranchType.BRANCH) + .setKey("master") + .setUuid(PROJECT.uuid()) + .setProjectUuid(PROJECT.uuid())); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public LogTester logTester = new LogTester(); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private ComponentRepositoryImpl componentRepository = new ComponentRepositoryImpl(); + private MutableMetricRepository metricRepository = new MutableMetricRepositoryImpl(); + private ProjectHolder projectHolder = mock(ProjectHolder.class); + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private ExportMeasuresStep underTest = new ExportMeasuresStep(dbTester.getDbClient(), projectHolder, componentRepository, metricRepository, dumpWriter); + + @Before + public void setUp() { + String projectUuid = dbTester.components().insertPublicProject(PROJECT).uuid(); + componentRepository.register(1, projectUuid, false); + dbTester.getDbClient().componentDao().insert(dbTester.getSession(), FILE, ANOTHER_PROJECT); + dbTester.getDbClient().metricDao().insert(dbTester.getSession(), NCLOC, DISABLED_METRIC); + dbTester.commit(); + when(projectHolder.projectDto()).thenReturn(dbTester.components().getProjectDto(PROJECT)); + when(projectHolder.branches()).thenReturn(BRANCHES); + } + + @Test + public void export_zero_measures() { + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.MEASURES)).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 measures exported"); + assertThat(metricRepository.getRefByUuid()).isEmpty(); + } + + @Test + public void export_measures() { + SnapshotDto firstAnalysis = insertSnapshot("U_1", PROJECT, STATUS_PROCESSED); + insertMeasure(firstAnalysis, PROJECT, new MeasureDto().setValue(100.0).setMetricUuid(NCLOC.getUuid())); + SnapshotDto secondAnalysis = insertSnapshot("U_2", PROJECT, STATUS_PROCESSED); + insertMeasure(secondAnalysis, PROJECT, new MeasureDto().setValue(110.0).setMetricUuid(NCLOC.getUuid())); + SnapshotDto anotherProjectAnalysis = insertSnapshot("U_3", ANOTHER_PROJECT, STATUS_PROCESSED); + insertMeasure(anotherProjectAnalysis, ANOTHER_PROJECT, new MeasureDto().setValue(500.0).setMetricUuid(NCLOC.getUuid())); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.MEASURES); + assertThat(exportedMeasures).hasSize(2); + assertThat(exportedMeasures).extracting(ProjectDump.Measure::getAnalysisUuid).containsOnly(firstAnalysis.getUuid(), secondAnalysis.getUuid()); + assertThat(exportedMeasures).extracting(ProjectDump.Measure::getMetricRef).containsOnly(0); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("2 measures exported"); + assertThat(metricRepository.getRefByUuid().keySet()).containsOnly(NCLOC.getUuid()); + } + + @Test + public void do_not_export_measures_on_unprocessed_snapshots() { + SnapshotDto firstAnalysis = insertSnapshot("U_1", PROJECT, STATUS_UNPROCESSED); + insertMeasure(firstAnalysis, PROJECT, new MeasureDto().setValue(100.0).setMetricUuid(NCLOC.getUuid())); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.MEASURES); + assertThat(exportedMeasures).isEmpty(); + } + + @Test + public void do_not_export_measures_on_disabled_metrics() { + SnapshotDto firstAnalysis = insertSnapshot("U_1", PROJECT, STATUS_PROCESSED); + insertMeasure(firstAnalysis, PROJECT, new MeasureDto().setValue(100.0).setMetricUuid(DISABLED_METRIC.getUuid())); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.MEASURES); + assertThat(exportedMeasures).isEmpty(); + } + + @Test + public void test_exported_fields() { + SnapshotDto analysis = insertSnapshot("U_1", PROJECT, STATUS_PROCESSED); + MeasureDto dto = new MeasureDto() + .setMetricUuid(NCLOC.getUuid()) + .setValue(100.0) + .setData("data") + .setAlertStatus("OK") + .setAlertText("alert text") + .setVariation(1.0); + insertMeasure(analysis, PROJECT, dto); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + List exportedMeasures = dumpWriter.getWrittenMessagesOf(DumpElement.MEASURES); + ProjectDump.Measure measure = exportedMeasures.get(0); + assertThat(measure.getAlertStatus()).isEqualTo(dto.getAlertStatus()); + assertThat(measure.getAlertText()).isEqualTo(dto.getAlertText()); + assertThat(measure.getDoubleValue().getValue()).isEqualTo(dto.getValue()); + assertThat(measure.getTextValue()).isEqualTo(dto.getData()); + assertThat(measure.getMetricRef()).isZero(); + assertThat(measure.getAnalysisUuid()).isEqualTo(analysis.getUuid()); + assertThat(measure.getVariation1().getValue()).isEqualTo(dto.getVariation()); + } + + @Test + public void test_null_exported_fields() { + SnapshotDto analysis = insertSnapshot("U_1", PROJECT, STATUS_PROCESSED); + insertMeasure(analysis, PROJECT, new MeasureDto().setMetricUuid(NCLOC.getUuid())); + dbTester.commit(); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Measure measure = dumpWriter.getWrittenMessagesOf(DumpElement.MEASURES).get(0); + assertThat(measure.getAlertStatus()).isEmpty(); + assertThat(measure.getAlertText()).isEmpty(); + assertThat(measure.hasDoubleValue()).isFalse(); + assertThat(measure.getTextValue()).isEmpty(); + assertThat(measure.hasVariation1()).isFalse(); + } + + @Test + public void test_getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Export measures"); + } + + private SnapshotDto insertSnapshot(String snapshotUuid, ComponentDto project, String status) { + SnapshotDto snapshot = new SnapshotDto() + .setUuid(snapshotUuid) + .setComponentUuid(project.uuid()) + .setStatus(status) + .setLast(true); + dbTester.getDbClient().snapshotDao().insert(dbTester.getSession(), snapshot); + return snapshot; + } + + private void insertMeasure(SnapshotDto analysisDto, ComponentDto componentDto, MeasureDto measureDto) { + measureDto + .setAnalysisUuid(analysisDto.getUuid()) + .setComponentUuid(componentDto.uuid()); + dbTester.getDbClient().measureDao().insert(dbTester.getSession(), measureDto); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStepTest.java new file mode 100644 index 00000000000..676fc0639c8 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportMetricsStepTest.java @@ -0,0 +1,129 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.common.base.Predicate; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import javax.annotation.Nonnull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.metric.MetricDto; + +import static com.google.common.collect.FluentIterable.from; +import static org.assertj.core.api.Assertions.assertThat; + +public class ExportMetricsStepTest { + + private static final MetricDto NCLOC = new MetricDto() + .setUuid("1") + .setKey("ncloc") + .setShortName("Lines of code") + .setEnabled(true); + private static final MetricDto COVERAGE = new MetricDto() + .setUuid("2") + .setKey("coverage") + .setShortName("Coverage") + .setEnabled(true); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public LogTester logTester = new LogTester(); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + MutableMetricRepository metricsHolder = new MutableMetricRepositoryImpl(); + FakeDumpWriter dumpWriter = new FakeDumpWriter(); + ExportMetricsStep underTest = new ExportMetricsStep(dbTester.getDbClient(), metricsHolder, dumpWriter); + + @Before + public void setUp() { + dbTester.getDbClient().metricDao().insert(dbTester.getSession(), NCLOC, COVERAGE); + dbTester.commit(); + } + + @Test + public void export_zero_metrics() { + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 metrics exported"); + } + + @Test + public void export_metrics() { + metricsHolder.add(NCLOC.getUuid()); + metricsHolder.add(COVERAGE.getUuid()); + + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("2 metrics exported"); + List exportedMetrics = dumpWriter.getWrittenMessagesOf(DumpElement.METRICS); + + ProjectDump.Metric ncloc = from(exportedMetrics).firstMatch(new HasMetricRefPredicate(0)).get(); + assertThat(ncloc.getRef()).isZero(); + assertThat(ncloc.getKey()).isEqualTo("ncloc"); + assertThat(ncloc.getName()).isEqualTo("Lines of code"); + + ProjectDump.Metric coverage = from(exportedMetrics).firstMatch(new HasMetricRefPredicate(1)).get(); + assertThat(coverage.getRef()).isEqualTo(1); + assertThat(coverage.getKey()).isEqualTo("coverage"); + assertThat(coverage.getName()).isEqualTo("Coverage"); + } + + @Test + public void throw_ISE_if_error() { + metricsHolder.add(NCLOC.getUuid()); + dumpWriter.failIfMoreThan(0, DumpElement.METRICS); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Metric Export failed after processing 0 metrics successfully"); + underTest.execute(new TestComputationStepContext()); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("Metric Export failed after processing 0 metrics successfully"); + } + + @Test + public void test_getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Export metrics"); + } + + private static class HasMetricRefPredicate implements Predicate { + private final int ref; + + HasMetricRefPredicate(int ref) { + this.ref = ref; + } + + @Override + public boolean apply(@Nonnull ProjectDump.Metric input) { + return input.getRef() == ref; + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStepTest.java new file mode 100644 index 00000000000..b171bff692a --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportNewCodePeriodsStepTest.java @@ -0,0 +1,178 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.common.collect.ImmutableList; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.Date; +import java.util.List; +import javax.annotation.Nullable; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.newcodeperiod.NewCodePeriodDto; +import org.sonar.db.newcodeperiod.NewCodePeriodType; +import org.sonar.db.project.ProjectExportMapper; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.PREVIOUS_VERSION; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.SPECIFIC_ANALYSIS; + +public class ExportNewCodePeriodsStepTest { + + private static final String PROJECT_UUID = "project_uuid"; + private static final String ANOTHER_PROJECT_UUID = "another_project_uuid"; + private static final ComponentDto PROJECT = new ComponentDto() + .setUuid(PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid(PROJECT_UUID) + .setProjectUuid(PROJECT_UUID) + .setQualifier(Qualifiers.PROJECT) + .setName("project") + .setDbKey("the_project"); + private static final ComponentDto ANOTHER_PROJECT = new ComponentDto() + .setUuid(ANOTHER_PROJECT_UUID) + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid(ANOTHER_PROJECT_UUID) + .setProjectUuid(ANOTHER_PROJECT_UUID) + .setQualifier(Qualifiers.PROJECT) + .setName("another_project") + .setDbKey("another_project"); + + private static final List PROJECT_BRANCHES = ImmutableList.of( + new BranchDto().setBranchType(BranchType.PULL_REQUEST).setProjectUuid(PROJECT_UUID).setKey("pr-1").setUuid("pr-uuid-1").setMergeBranchUuid("master"), + new BranchDto().setBranchType(BranchType.BRANCH).setProjectUuid(PROJECT_UUID).setKey("branch-1").setUuid("branch-uuid-1").setMergeBranchUuid("master") + .setExcludeFromPurge(true), + new BranchDto().setBranchType(BranchType.BRANCH).setProjectUuid(PROJECT_UUID).setKey("branch-2").setUuid("branch-uuid-2").setMergeBranchUuid("master") + .setExcludeFromPurge(false)); + + private static final List ANOTHER_PROJECT_BRANCHES = ImmutableList.of( + new BranchDto().setBranchType(BranchType.BRANCH).setProjectUuid(ANOTHER_PROJECT_UUID).setKey("branch-3").setUuid("branch-uuid-3").setMergeBranchUuid("master") + .setExcludeFromPurge(true)); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public LogTester logTester = new LogTester(); + @Rule + public DbTester dbTester = DbTester.createWithExtensionMappers(System2.INSTANCE, ProjectExportMapper.class); + + private MutableProjectHolder projectHolder = new MutableProjectHolderImpl(); + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private ExportNewCodePeriodsStep underTest = new ExportNewCodePeriodsStep(dbTester.getDbClient(), projectHolder, dumpWriter); + + @Before + public void setUp() { + Date createdAt = new Date(); + ComponentDto projectDto = dbTester.components().insertPublicProject(PROJECT); + PROJECT_BRANCHES.forEach(branch -> dbTester.components().insertProjectBranch(projectDto, branch).setCreatedAt(createdAt)); + + ComponentDto anotherProjectDto = dbTester.components().insertPublicProject(ANOTHER_PROJECT); + ANOTHER_PROJECT_BRANCHES.forEach(branch -> dbTester.components().insertProjectBranch(anotherProjectDto, branch).setCreatedAt(createdAt)); + + dbTester.commit(); + projectHolder.setProjectDto(dbTester.components().getProjectDto(PROJECT)); + } + + @Test + public void export_zero_new_code_periods() { + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.NEW_CODE_PERIODS)).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 new code periods exported"); + } + + @Test + public void export_only_project_new_code_periods_on_branches_excluded_from_purge() { + NewCodePeriodDto newCodePeriod1 = newDto("uuid1", PROJECT.uuid(), null, SPECIFIC_ANALYSIS, "analysis-uuid"); + NewCodePeriodDto newCodePeriod2 = newDto("uuid2", PROJECT.uuid(), "branch-uuid-1", SPECIFIC_ANALYSIS, "analysis-uuid"); + // the following new code periods are not exported + NewCodePeriodDto newCodePeriod3 = newDto("uuid3", PROJECT.uuid(), "branch-uuid-2", SPECIFIC_ANALYSIS, "analysis-uuid"); + NewCodePeriodDto anotherProjectNewCodePeriods = newDto("uuid4", ANOTHER_PROJECT.uuid(), "branch-uuid-3", SPECIFIC_ANALYSIS, "analysis-uuid"); + NewCodePeriodDto globalNewCodePeriod = newDto("uuid5", null, null, PREVIOUS_VERSION, null); + insertNewCodePeriods(newCodePeriod1, newCodePeriod2, newCodePeriod3, anotherProjectNewCodePeriods, globalNewCodePeriod); + + underTest.execute(new TestComputationStepContext()); + + List exportedProps = dumpWriter.getWrittenMessagesOf(DumpElement.NEW_CODE_PERIODS); + assertThat(exportedProps).hasSize(2); + assertThat(exportedProps).extracting(ProjectDump.NewCodePeriod::getUuid).containsOnly("uuid1", "uuid2"); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("2 new code periods exported"); + } + + @Test + public void test_exported_fields() { + NewCodePeriodDto dto = newDto("uuid1", PROJECT.uuid(), "branch-uuid-1", SPECIFIC_ANALYSIS, "analysis-uuid"); + insertNewCodePeriods(dto); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.NewCodePeriod exportedNewCodePeriod = dumpWriter.getWrittenMessagesOf(DumpElement.NEW_CODE_PERIODS).get(0); + assertThat(exportedNewCodePeriod.getUuid()).isEqualTo(dto.getUuid()); + assertThat(exportedNewCodePeriod.getProjectUuid()).isEqualTo(dto.getProjectUuid()); + assertThat(exportedNewCodePeriod.getBranchUuid()).isEqualTo(dto.getBranchUuid()); + assertThat(exportedNewCodePeriod.getType()).isEqualTo(dto.getType().name()); + assertThat(exportedNewCodePeriod.getValue()).isEqualTo(dto.getValue()); + } + + @Test + public void throws_ISE_if_error() { + dumpWriter.failIfMoreThan(1, DumpElement.NEW_CODE_PERIODS); + insertNewCodePeriods( + newDto("uuid1", PROJECT.uuid(), null, SPECIFIC_ANALYSIS, "analysis-uuid"), + newDto("uuid2", PROJECT.uuid(), "branch-uuid-1", SPECIFIC_ANALYSIS, "analysis-uuid")); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("New Code Periods Export failed after processing 1 new code periods successfully"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void test_getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Export new code periods"); + } + + private static NewCodePeriodDto newDto(String uuid, @Nullable String projectUuid, @Nullable String branchUuid, NewCodePeriodType type, @Nullable String value) { + return new NewCodePeriodDto() + .setUuid(uuid) + .setProjectUuid(projectUuid) + .setBranchUuid(branchUuid) + .setType(type) + .setValue(value); + } + + private void insertNewCodePeriods(NewCodePeriodDto... dtos) { + for (NewCodePeriodDto dto : dtos) { + dbTester.getDbClient().newCodePeriodDao().insert(dbTester.getSession(), dto); + } + dbTester.commit(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStepTest.java new file mode 100644 index 00000000000..86d8e4aa65b --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportPluginsStepTest.java @@ -0,0 +1,103 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.core.platform.PluginInfo; +import org.sonar.core.platform.PluginRepository; +import org.sonar.updatecenter.common.Version; + +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ExportPluginsStepTest { + + @Rule + public LogTester logTester = new LogTester(); + + private PluginRepository pluginRepository = mock(PluginRepository.class); + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + ExportPluginsStep underTest = new ExportPluginsStep(pluginRepository, dumpWriter); + + @Test + public void export_plugins() { + when(pluginRepository.getPluginInfos()).thenReturn(Arrays.asList( + new PluginInfo("java"), new PluginInfo("cs"))); + + underTest.execute(new TestComputationStepContext()); + + List exportedPlugins = dumpWriter.getWrittenMessagesOf(DumpElement.PLUGINS); + assertThat(exportedPlugins).hasSize(2); + assertThat(exportedPlugins).extracting(ProjectDump.Plugin::getKey).containsExactlyInAnyOrder("java", "cs"); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("2 plugins exported"); + } + + @Test + public void export_zero_plugins() { + when(pluginRepository.getPluginInfos()).thenReturn(Collections.emptyList()); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.PLUGINS)).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 plugins exported"); + } + + @Test + public void test_exported_fields() { + when(pluginRepository.getPluginInfos()).thenReturn(singletonList( + new PluginInfo("java").setName("Java").setVersion(Version.create("1.2.3")))); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Plugin exportedPlugin = dumpWriter.getWrittenMessagesOf(DumpElement.PLUGINS).get(0); + assertThat(exportedPlugin.getKey()).isEqualTo("java"); + assertThat(exportedPlugin.getName()).isEqualTo("Java"); + assertThat(exportedPlugin.getVersion()).isEqualTo("1.2.3"); + } + + @Test + public void test_nullable_exported_fields() { + when(pluginRepository.getPluginInfos()).thenReturn(singletonList( + new PluginInfo("java"))); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Plugin exportedPlugin = dumpWriter.getWrittenMessagesOf(DumpElement.PLUGINS).get(0); + assertThat(exportedPlugin.getKey()).isEqualTo("java"); + // if name is not set, then value is the same as key + assertThat(exportedPlugin.getName()).isEqualTo("java"); + assertThat(exportedPlugin.getVersion()).isEmpty(); + } + + @Test + public void test_getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Export plugins"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStepTest.java new file mode 100644 index 00000000000..4c25a186542 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/ExportSettingsStepTest.java @@ -0,0 +1,170 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.List; +import javax.annotation.Nullable; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.task.projectexport.component.ComponentRepositoryImpl; +import org.sonar.ce.task.projectexport.component.MutableComponentRepository; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.project.ProjectExportMapper; +import org.sonar.db.property.PropertyDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.component.ComponentDto.UUID_PATH_OF_ROOT; + +public class ExportSettingsStepTest { + + private static final ComponentDto GLOBAL = null; + private static final ComponentDto PROJECT = new ComponentDto() + .setUuid("project_uuid") + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid("project_uuid") + .setProjectUuid("project_uuid") + .setDbKey("the_project"); + private static final ComponentDto ANOTHER_PROJECT = new ComponentDto() + .setUuid("another_project_uuid") + .setUuidPath(UUID_PATH_OF_ROOT) + .setRootUuid("another_project_uuid") + .setProjectUuid("another_project_uuid") + .setDbKey("another_project"); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public LogTester logTester = new LogTester(); + @Rule + public DbTester dbTester = DbTester.createWithExtensionMappers(System2.INSTANCE, ProjectExportMapper.class); + private MutableComponentRepository componentRepository = new ComponentRepositoryImpl(); + private MutableProjectHolder projectHolder = new MutableProjectHolderImpl(); + private FakeDumpWriter dumpWriter = new FakeDumpWriter(); + private ExportSettingsStep underTest = new ExportSettingsStep(dbTester.getDbClient(), projectHolder, componentRepository, dumpWriter); + + @Before + public void setUp() { + dbTester.components().insertPublicProject(PROJECT); + dbTester.components().insertPublicProject(ANOTHER_PROJECT); + dbTester.commit(); + projectHolder.setProjectDto(dbTester.components().getProjectDto(PROJECT)); + componentRepository.register(1, PROJECT.uuid(), false); + } + + @Test + public void export_zero_settings() { + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.SETTINGS)).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 settings exported"); + } + + @Test + public void export_only_project_settings() { + PropertyDto projectProperty1 = newDto("p1", "v1", PROJECT); + PropertyDto projectProperty2 = newDto("p2", "v2", PROJECT); + // the following properties are not exported + PropertyDto propOnOtherProject = newDto("p3", "v3", ANOTHER_PROJECT); + PropertyDto globalProperty = newDto("p4", "v4", GLOBAL); + insertProperties(PROJECT.getKey(), PROJECT.name(), projectProperty1, projectProperty2); + insertProperties(ANOTHER_PROJECT.getKey(), ANOTHER_PROJECT.name(), propOnOtherProject); + insertProperties(null, null, globalProperty); + + underTest.execute(new TestComputationStepContext()); + + List exportedProps = dumpWriter.getWrittenMessagesOf(DumpElement.SETTINGS); + assertThat(exportedProps).hasSize(2); + assertThat(exportedProps).extracting(ProjectDump.Setting::getKey).containsOnly("p1", "p2"); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("2 settings exported"); + } + + @Test + public void exclude_properties_specific_to_environment() { + insertProperties(PROJECT.getKey(), PROJECT.name(), newDto("sonar.issues.defaultAssigneeLogin", null, PROJECT)); + + underTest.execute(new TestComputationStepContext()); + + assertThat(dumpWriter.getWrittenMessagesOf(DumpElement.SETTINGS)).isEmpty(); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("0 settings exported"); + } + + @Test + public void test_exported_fields() { + PropertyDto dto = newDto("p1", "v1", PROJECT); + insertProperties(PROJECT.getKey(), PROJECT.name(), dto); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Setting exportedProp = dumpWriter.getWrittenMessagesOf(DumpElement.SETTINGS).get(0); + assertThat(exportedProp.getKey()).isEqualTo(dto.getKey()); + assertThat(exportedProp.getValue()).isEqualTo(dto.getValue()); + } + + @Test + public void property_can_have_empty_value() { + insertProperties(PROJECT.getKey(), PROJECT.name(), newDto("p1", null, PROJECT)); + + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Setting exportedProp = dumpWriter.getWrittenMessagesOf(DumpElement.SETTINGS).get(0); + assertThat(exportedProp.getKey()).isEqualTo("p1"); + assertThat(exportedProp.getValue()).isEmpty(); + } + + @Test + public void throws_ISE_if_error() { + dumpWriter.failIfMoreThan(1, DumpElement.SETTINGS); + insertProperties(PROJECT.getKey(), PROJECT.name(), newDto("p1", null, PROJECT), + newDto("p2", null, PROJECT)); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Settings Export failed after processing 1 settings successfully"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void test_getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Export settings"); + } + + private static PropertyDto newDto(String key, @Nullable String value, @Nullable ComponentDto project) { + PropertyDto dto = new PropertyDto().setKey(key).setValue(value); + if (project != null) { + dto.setComponentUuid(project.uuid()); + } + return dto; + } + + private void insertProperties(@Nullable String componentKey, @Nullable String componentName, PropertyDto... dtos) { + for (PropertyDto dto : dtos) { + dbTester.getDbClient().propertiesDao().saveProperty(dbTester.getSession(), dto, null, componentKey, componentName, Qualifiers.VIEW); + } + dbTester.commit(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/FakeDumpWriter.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/FakeDumpWriter.java new file mode 100644 index 00000000000..35829ca641b --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/FakeDumpWriter.java @@ -0,0 +1,115 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.google.common.base.MoreObjects; +import com.google.protobuf.Message; +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static org.sonar.ce.task.projectexport.steps.DumpElement.METADATA; + +public class FakeDumpWriter implements DumpWriter { + + private boolean published = false; + private final Map, ListStreamWriter> writersByDumpElement = new HashMap<>(); + private final Map, Integer> failureThresholds = new HashMap<>(); + + @Override + public void write(ProjectDump.Metadata metadata) { + checkNotPublished(); + newStreamWriter(METADATA).write(metadata); + } + + /** + * @throws IllegalStateException if metadata have not been written yet. + * @see #write(ProjectDump.Metadata) + */ + public ProjectDump.Metadata getMetadata() { + return getWrittenMessagesOf(METADATA).get(0); + } + + /** + * @throws IllegalStateException if messages have not been written yet for the specified {@code DumpElement} + * @see #newStreamWriter(DumpElement) + */ + @SuppressWarnings("unchecked") + public List getWrittenMessagesOf(DumpElement elt) { + ListStreamWriter writer = (ListStreamWriter) writersByDumpElement.get(elt); + checkState(writer != null); + return writer.messages; + } + + @Override + public StreamWriter newStreamWriter(DumpElement elt) { + checkNotPublished(); + checkState(!writersByDumpElement.containsKey(elt)); + + int failureThreshold = MoreObjects.firstNonNull(failureThresholds.get(elt), Integer.MAX_VALUE); + ListStreamWriter writer = new ListStreamWriter<>(failureThreshold); + writersByDumpElement.put(elt, writer); + return writer; + } + + /** + * The stream returned by {@link #newStreamWriter(DumpElement)} will throw an + * {@link IllegalStateException} if more than {@code count} messages are written. + * By default no exception is thrown. + */ + public void failIfMoreThan(int count, DumpElement element) { + failureThresholds.put(element, count); + } + + @Override + public void publish() { + checkNotPublished(); + published = true; + } + + private void checkNotPublished() { + checkState(!published, "Dump is already published"); + } + + private static class ListStreamWriter implements StreamWriter { + private final List messages = new ArrayList<>(); + private final int failureThreshold; + + private ListStreamWriter(int failureThreshold) { + checkArgument(failureThreshold >= 0, "Threshold (%d) must be positive", failureThreshold); + this.failureThreshold = failureThreshold; + } + + @Override + public void write(MSG msg) { + checkState(messages.size() < failureThreshold, "Maximum of %d written messages has been reached", failureThreshold); + messages.add(msg); + } + + @Override + public void close() { + // nothing to do + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/LoadProjectStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/LoadProjectStepTest.java new file mode 100644 index 00000000000..4aa0d6b55c5 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/LoadProjectStepTest.java @@ -0,0 +1,82 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.MessageException; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; + +import static org.assertj.core.api.Assertions.assertThat; + +public class LoadProjectStepTest { + + private static final String PROJECT_KEY = "project_key"; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private final ProjectDescriptor descriptor = new ProjectDescriptor("project_uuid", PROJECT_KEY, "Project Name"); + private final MutableProjectHolder definitionHolder = new MutableProjectHolderImpl(); + private final LoadProjectStep underTest = new LoadProjectStep(descriptor, definitionHolder, dbTester.getDbClient()); + + @Test + public void fails_if_project_does_not_exist() { + expectedException.expect(MessageException.class); + expectedException.expectMessage("Project with key [project_key] does not exist"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void fails_if_component_is_not_a_project() { + // insert a module, but not a project + dbTester.executeInsert("projects", + "kee", PROJECT_KEY, + "qualifier", Qualifiers.APP, + "uuid", "not_used", + "private", false, + "created_at", 1L, + "updated_at", 1L); + + expectedException.expect(MessageException.class); + expectedException.expectMessage("Project with key [project_key] does not exist"); + underTest.execute(new TestComputationStepContext()); + } + + @Test + public void registers_project_if_valid() { + ComponentDto project = dbTester.components().insertPublicProject(c -> c.setDbKey(PROJECT_KEY)); + underTest.execute(new TestComputationStepContext()); + assertThat(definitionHolder.projectDto().getKey()).isEqualTo(project.getKey()); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isNotEmpty(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImplTest.java new file mode 100644 index 00000000000..91bf7924adc --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/MutableMetricRepositoryImplTest.java @@ -0,0 +1,51 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +public class MutableMetricRepositoryImplTest { + + MutableMetricRepository underTest = new MutableMetricRepositoryImpl(); + + @Test + public void add_ref() { + underTest.add("10"); + underTest.add("12"); + + assertThat(underTest.getRefByUuid().entrySet()).containsOnly(entry("10", 0), entry("12", 1)); + } + + @Test + public void add_multiple_times_the_same_ref() { + underTest.add("10"); + underTest.add("10"); + + assertThat(underTest.getRefByUuid().entrySet()).containsExactly(entry("10", 0)); + } + + @Test + public void getAll_returns_empty_set() { + assertThat(underTest.getRefByUuid()).isEmpty(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/PublishDumpStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/PublishDumpStepTest.java new file mode 100644 index 00000000000..7a0e8d3cdcb --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/PublishDumpStepTest.java @@ -0,0 +1,43 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import org.junit.Test; +import org.sonar.ce.task.step.TestComputationStepContext; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +public class PublishDumpStepTest { + DumpWriter dumpWriter = mock(DumpWriter.class); + PublishDumpStep underTest = new PublishDumpStep(dumpWriter); + + @Test + public void archives_dump() { + underTest.execute(new TestComputationStepContext()); + verify(dumpWriter).publish(); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isEqualTo("Publish dump file"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStepTest.java new file mode 100644 index 00000000000..2bea4b0187f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/steps/WriteMetadataStepTest.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.steps; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump; +import org.junit.Test; +import org.sonar.api.SonarEdition; +import org.sonar.api.SonarQubeSide; +import org.sonar.api.internal.SonarRuntimeImpl; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.Version; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.project.ProjectDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class WriteMetadataStepTest { + + private static final String PROJECT_KEY = "project_key"; + private static final String PROJECT_UUID = "project_uuid"; + private static final long NOW = 123L; + + System2 system2 = spy(System2.INSTANCE); + FakeDumpWriter dumpWriter = new FakeDumpWriter(); + MutableProjectHolderImpl projectHolder = new MutableProjectHolderImpl(); + Version sqVersion = Version.create(6, 0); + WriteMetadataStep underTest = new WriteMetadataStep(system2, dumpWriter, projectHolder, + SonarRuntimeImpl.forSonarQube(sqVersion, SonarQubeSide.SERVER, SonarEdition.COMMUNITY)); + + @Test + public void write_metadata() { + when(system2.now()).thenReturn(NOW); + ProjectDto dto = new ProjectDto().setKey(PROJECT_KEY).setUuid(PROJECT_UUID); + projectHolder.setProjectDto(dto); + underTest.execute(new TestComputationStepContext()); + + ProjectDump.Metadata metadata = dumpWriter.getMetadata(); + assertThat(metadata.getProjectKey()).isEqualTo(PROJECT_KEY); + assertThat(metadata.getProjectUuid()).isEqualTo(PROJECT_UUID); + assertThat(metadata.getSonarqubeVersion()).isEqualTo(sqVersion.toString()); + assertThat(metadata.getDumpDate()).isEqualTo(NOW); + } + + @Test + public void getDescription_is_defined() { + assertThat(underTest.getDescription()).isNotEmpty(); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImplTest.java new file mode 100644 index 00000000000..64752d4589f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectExportDumpFSImplTest.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.util; + +import java.io.File; +import java.io.IOException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ProjectExportDumpFSImplTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private final MapSettings settings = new MapSettings(); + private final ProjectDescriptor projectDescriptor = new ProjectDescriptor("uuid", "project_key", "name"); + private final ProjectDescriptor descriptorWithUglyKey = new ProjectDescriptor("uuid", " so:me à9ç& Key ", "name"); + + private File dataDir; + private ProjectExportDumpFSImpl underTest; + + @Before + public void setUp() throws Exception { + this.dataDir = temp.newFolder(); + settings.setProperty("sonar.path.data", dataDir.getAbsolutePath()); + this.underTest = new ProjectExportDumpFSImpl(settings.asConfig()); + } + + @Test + public void start_creates_import_and_export_directories_including_missing_parents() throws IOException { + dataDir = new File(temp.newFolder(), "data"); + File importDir = new File(dataDir, "governance/project_dumps/import"); + File exportDir = new File(dataDir, "governance/project_dumps/export"); + + settings.setProperty("sonar.path.data", dataDir.getAbsolutePath()); + this.underTest = new ProjectExportDumpFSImpl(settings.asConfig()); + + assertThat(dataDir).doesNotExist(); + assertThat(importDir).doesNotExist(); + assertThat(exportDir).doesNotExist(); + + underTest.start(); + + assertThat(dataDir).exists().isDirectory(); + assertThat(importDir).exists().isDirectory(); + assertThat(exportDir).exists().isDirectory(); + } + + @Test + public void exportDumpOf_is_located_in_governance_project_dump_out() { + assertThat(underTest.exportDumpOf(projectDescriptor)).isEqualTo(new File(dataDir, "governance/project_dumps/export/project_key.zip")); + } + + @Test + public void exportDumpOf_slugifies_project_key() { + assertThat(underTest.exportDumpOf(descriptorWithUglyKey)) + .isEqualTo(new File(dataDir, "governance/project_dumps/export/so-me-a9c-key.zip")); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImplTest.java new file mode 100644 index 00000000000..cd12341368e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectexport/util/ProjectImportDumpFSImplTest.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectexport.util; + +import java.io.File; +import java.io.IOException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectDescriptor; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ProjectImportDumpFSImplTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private final MapSettings settings = new MapSettings(); + private final ProjectDescriptor projectDescriptor = new ProjectDescriptor("uuid", "project_key", "name"); + private final ProjectDescriptor descriptorWithUglyKey = new ProjectDescriptor("uuid", " so:me à9ç& Key ", "name"); + + private File dataDir; + private ProjectImportDumpFSImpl underTest; + + @Before + public void setUp() throws Exception { + this.dataDir = temp.newFolder(); + settings.setProperty("sonar.path.data", dataDir.getAbsolutePath()); + this.underTest = new ProjectImportDumpFSImpl(settings.asConfig()); + } + + @Test + public void start_creates_import_and_export_directories_including_missing_parents() throws IOException { + dataDir = new File(temp.newFolder(), "data"); + File importDir = new File(dataDir, "governance/project_dumps/import"); + File exportDir = new File(dataDir, "governance/project_dumps/export"); + + settings.setProperty("sonar.path.data", dataDir.getAbsolutePath()); + this.underTest = new ProjectImportDumpFSImpl(settings.asConfig()); + + assertThat(dataDir).doesNotExist(); + assertThat(importDir).doesNotExist(); + assertThat(exportDir).doesNotExist(); + + underTest.start(); + + assertThat(dataDir).exists().isDirectory(); + assertThat(importDir).exists().isDirectory(); + assertThat(exportDir).exists().isDirectory(); + } + + @Test + public void importDumpOf_is_located_in_governance_project_dump_in() { + assertThat(underTest.importDumpOf(projectDescriptor)).isEqualTo(new File(dataDir, "governance/project_dumps/import/project_key.zip")); + } + + @Test + public void importDumpOf_slugifies_project_key() { + assertThat(underTest.importDumpOf(descriptorWithUglyKey)) + .isEqualTo(new File(dataDir, "governance/project_dumps/import/so-me-a9c-key.zip")); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Files2Test.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Files2Test.java new file mode 100644 index 00000000000..26461d03bb9 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Files2Test.java @@ -0,0 +1,252 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; + +public class Files2Test { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + Files2 underTest = spy(Files2.FILES2); + + @Test + public void deleteIfExists_does_nothing_if_file_does_not_exist() throws Exception { + File file = temp.newFile(); + assertThat(file.delete()).isTrue(); + + underTest.deleteIfExists(file); + assertThat(file).doesNotExist(); + } + + @Test + public void deleteIfExists_deletes_directory() throws Exception { + File dir = temp.newFolder(); + + underTest.deleteIfExists(dir); + assertThat(dir).doesNotExist(); + } + + @Test + public void deleteIfExists_deletes_file() throws Exception { + File file = temp.newFile(); + + underTest.deleteIfExists(file); + assertThat(file).doesNotExist(); + } + + @Test + public void deleteIfExists_throws_ISE_on_error() throws Exception { + File file = temp.newFile(); + doThrow(new IOException("failure")).when(underTest).deleteIfExistsOrThrowIOE(file); + + expectedException.expect(IllegalStateException.class); + underTest.deleteIfExists(file); + } + + @Test + public void openInputStream_opens_existing_file() throws Exception { + File file = temp.newFile(); + FileUtils.write(file, "foo"); + + try (FileInputStream input = underTest.openInputStream(file)) { + assertThat(IOUtils.toString(input)).isEqualTo("foo"); + } + } + + @Test + public void openInputStream_throws_ISE_if_file_does_not_exist() throws Exception { + final File file = temp.newFile(); + assertThat(file.delete()).isTrue(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not open file " + file); + expectCauseMessage("File " + file + " does not exist"); + underTest.openInputStream(file); + } + + @Test + public void openInputStream_throws_ISE_if_file_is_a_directory() throws Exception { + File dir = temp.newFolder(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not open file " + dir); + expectCauseMessage("File " + dir + " exists but is a directory"); + underTest.openInputStream(dir); + } + + @Test + public void openOutputStream_creates_file() throws Exception { + File file = temp.newFile(); + assertThat(file.delete()).isTrue(); + + try (FileOutputStream outputStream = underTest.openOutputStream(file, false)) { + IOUtils.write("foo", outputStream); + } + assertThat(FileUtils.readFileToString(file)).isEqualTo("foo"); + } + + @Test + public void openOutputStream_appends_bytes_to_existing_file() throws Exception { + File file = temp.newFile(); + FileUtils.write(file, "foo"); + + try (FileOutputStream outputStream = underTest.openOutputStream(file, true)) { + IOUtils.write("bar", outputStream); + } + assertThat(FileUtils.readFileToString(file)).isEqualTo("foobar"); + } + + @Test + public void openOutputStream_overwrites_existing_file() throws Exception { + File file = temp.newFile(); + FileUtils.write(file, "foo"); + + try (FileOutputStream outputStream = underTest.openOutputStream(file, false)) { + IOUtils.write("bar", outputStream); + } + assertThat(FileUtils.readFileToString(file)).isEqualTo("bar"); + } + + @Test + public void openOutputStream_throws_ISE_if_file_is_a_directory() throws Exception { + File dir = temp.newFolder(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not open file " + dir); + expectCauseMessage("File " + dir + " exists but is a directory"); + underTest.openOutputStream(dir, false); + } + + @Test + public void zipDir_an_existing_directory_then_unzipToDir() throws Exception { + File dir = temp.newFolder(); + FileUtils.write(new File(dir, "foo.txt"), "foo"); + + File zipFile = temp.newFile(); + underTest.zipDir(dir, zipFile); + assertThat(zipFile).exists().isFile(); + + File unzippedDir = temp.newFolder(); + underTest.unzipToDir(zipFile, unzippedDir); + assertThat(FileUtils.readFileToString(new File(unzippedDir, "foo.txt"))).isEqualTo("foo"); + } + + @Test + public void zipDir_throws_ISE_if_directory_does_not_exist() throws Exception { + File dir = temp.newFolder(); + underTest.deleteIfExists(dir); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not zip directory " + dir); + expectCauseMessage("Directory " + dir + " does not exist"); + underTest.zipDir(dir, temp.newFile()); + } + + @Test + public void zipDir_throws_ISE_if_directory_is_a_file() throws Exception { + File file = temp.newFile(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not zip directory " + file); + expectCauseMessage("File " + file + " exists but is not a directory"); + underTest.zipDir(file, temp.newFile()); + } + + @Test + public void createDir_creates_specified_directory() throws IOException { + File dir = new File(temp.newFolder(), "someDir"); + + assertThat(dir).doesNotExist(); + + underTest.createDir(dir); + + assertThat(dir).exists(); + } + + @Test + public void createDir_creates_specified_directory_and_missing_parents() throws IOException { + File dir1 = new File(temp.newFolder(), "dir1"); + File dir2 = new File(dir1, "dir2"); + File dir = new File(dir2, "someDir"); + + assertThat(dir1).doesNotExist(); + assertThat(dir2).doesNotExist(); + assertThat(dir).doesNotExist(); + + underTest.createDir(dir); + + assertThat(dir1).exists(); + assertThat(dir2).exists(); + assertThat(dir).exists(); + } + + @Test + public void createDir_throws_ISE_if_File_is_an_existing_file() throws IOException { + File file = temp.newFile(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage(file.toPath() + " is not a directory"); + + underTest.createDir(file); + } + + @Test + public void createDir_throws_ISE_if_File_is_an_existing_link() throws IOException { + File file = Files.createLink(new File(temp.newFolder(), "toto.lnk").toPath(), temp.newFile().toPath()).toFile(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage(file.toPath() + " is not a directory"); + + underTest.createDir(file); + } + + private void expectCauseMessage(final String msg) { + expectedException.expectCause(new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(Throwable throwable) { + return throwable.getMessage().contains(msg); + } + + @Override + public void describeTo(Description description) { + } + }); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Protobuf2Test.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Protobuf2Test.java new file mode 100644 index 00000000000..741e91a021f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/util/Protobuf2Test.java @@ -0,0 +1,129 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.util; + +import com.sonarsource.governance.projectdump.protobuf.ProjectDump.Metadata; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.ce.task.util.Files2.FILES2; + +public class Protobuf2Test { + + private static final String PROJECT_KEY_1 = "foo"; + private static final String PROJECT_KEY_2 = "bar"; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + Protobuf2 underTest = Protobuf2.PROTOBUF2; + + @Test + public void write_to_and_parse_from_file() throws Exception { + File file = temp.newFile(); + try (FileOutputStream output = FILES2.openOutputStream(file, false)) { + underTest.writeTo(newMetadata(PROJECT_KEY_1), output); + } + try (FileInputStream input = FILES2.openInputStream(file)) { + Metadata metadata = underTest.parseFrom(Metadata.parser(), input); + assertThat(metadata.getProjectKey()).isEqualTo(PROJECT_KEY_1); + } + } + + @Test + public void write_to_and_parse_delimited_from_file() throws Exception { + File file = temp.newFile(); + try (FileOutputStream output = FILES2.openOutputStream(file, false)) { + underTest.writeDelimitedTo(newMetadata(PROJECT_KEY_1), output); + underTest.writeDelimitedTo(newMetadata(PROJECT_KEY_2), output); + } + try (FileInputStream input = FILES2.openInputStream(file)) { + assertThat(underTest.parseDelimitedFrom(Metadata.parser(), input).getProjectKey()).isEqualTo(PROJECT_KEY_1); + assertThat(underTest.parseDelimitedFrom(Metadata.parser(), input).getProjectKey()).isEqualTo(PROJECT_KEY_2); + assertThat(underTest.parseDelimitedFrom(Metadata.parser(), input)).isNull(); + } + } + + @Test + public void writeTo_throws_ISE_on_error() throws Exception { + try (FailureOutputStream output = new FailureOutputStream()) { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not write message"); + underTest.writeTo(newMetadata(PROJECT_KEY_1), output); + } + } + + @Test + public void writeDelimitedTo_throws_ISE_on_error() throws Exception { + try (FailureOutputStream output = new FailureOutputStream()) { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not write message"); + underTest.writeDelimitedTo(newMetadata(PROJECT_KEY_1), output); + } + } + + @Test + public void parseFrom_throws_ISE_on_error() throws Exception { + try (FailureInputStream input = new FailureInputStream()) { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not parse message"); + underTest.parseFrom(Metadata.parser(), input); + } + } + + @Test + public void parseDelimitedFrom_throws_ISE_on_error() throws Exception { + try (FailureInputStream input = new FailureInputStream()) { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Can not parse message"); + underTest.parseDelimitedFrom(Metadata.parser(), input); + } + } + + private static Metadata newMetadata(String projectKey) { + return Metadata.newBuilder().setProjectKey(projectKey).build(); + } + + private static class FailureOutputStream extends OutputStream { + @Override + public void write(int b) throws IOException { + throw new IOException("Failure"); + } + } + + private static class FailureInputStream extends InputStream { + @Override + public int read() throws IOException { + throw new IOException("failure"); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/resources/org/sonar/ce/task/projectexport/issue/rubbish_data.txt b/server/sonar-ce-task-projectanalysis/src/test/resources/org/sonar/ce/task/projectexport/issue/rubbish_data.txt new file mode 100644 index 00000000000..3b5aee32cc5 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/resources/org/sonar/ce/task/projectexport/issue/rubbish_data.txt @@ -0,0 +1,3 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas hendrerit sapien magna, eget ultrices sapien ornare et. Donec interdum posuere magna nec tristique. Nam a dictum diam. Fusce consequat dolor vel justo egestas, a vulputate urna mattis. Nam at velit a quam eleifend aliquam quis vel est. Praesent facilisis consequat magna, eget aliquam leo eleifend non. Nunc et posuere tortor, sit amet tincidunt metus. Donec bibendum ligula id nisi lobortis, interdum vestibulum neque dignissim. Phasellus ornare sapien at auctor posuere. In hac habitasse platea dictumst. Sed nec tempor purus, sed interdum erat. + +Etiam nec porta magna, eget efficitur augue. Maecenas commodo ac nibh a dignissim. Curabitur eu ipsum nibh. Aenean id sodales velit. Vestibulum tristique at lectus eu gravida. Suspendisse vehicula gravida elit, eget fringilla nibh ullamcorper vitae. Integer est neque, aliquet sit amet turpis eget, tempor tincidunt nisi. Cras ornare mollis lacinia. Aenean at dolor scelerisque, rhoncus ex eget, pellentesque justo. Proin at malesuada ex. Praesent feugiat sapien id mollis vulputate. Fusce ornare, ligula id auctor elementum, enim erat finibus velit, vitae auctor metus ante vitae mauris. Quisque efficitur arcu vel enim sollicitudin accumsan. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Donec gravida erat vitae laoreet posuere. diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java index dfb93d67616..6ef83f82a46 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java @@ -101,6 +101,7 @@ import org.sonar.db.portfolio.PortfolioMapper; import org.sonar.db.portfolio.PortfolioProjectDto; import org.sonar.db.portfolio.PortfolioReferenceDto; import org.sonar.db.project.ProjectDto; +import org.sonar.db.project.ProjectExportMapper; import org.sonar.db.project.ProjectMapper; import org.sonar.db.property.InternalComponentPropertiesMapper; import org.sonar.db.property.InternalComponentPropertyDto; @@ -271,6 +272,7 @@ public class MyBatis implements Startable { ProjectAlmSettingMapper.class, ProjectLinkMapper.class, ProjectMapper.class, + ProjectExportMapper.class, ProjectMappingsMapper.class, ProjectQgateAssociationMapper.class, PropertiesMapper.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java index aae74db3d32..8561950b052 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java @@ -24,6 +24,7 @@ public final class CeTaskTypes { public static final String AUDIT_PURGE = "AUDIT_PURGE"; public static final String BRANCH_ISSUE_SYNC = "ISSUE_SYNC"; public static final String REPORT = "REPORT"; + public static final String PROJECT_EXPORT = "PROJECT_EXPORT"; private CeTaskTypes() { // only statics diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java new file mode 100644 index 00000000000..d398819cf55 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java @@ -0,0 +1,38 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.project; + +import java.util.List; +import org.apache.ibatis.annotations.Param; +import org.sonar.db.component.BranchDto; +import org.sonar.db.component.ProjectLinkDto; +import org.sonar.db.newcodeperiod.NewCodePeriodDto; +import org.sonar.db.property.PropertyDto; + +public interface ProjectExportMapper { + + List selectBranchesForExport(@Param("projectUuid") String projectUuid); + + List selectPropertiesForExport(@Param("projectUuid") String projectUuid); + + List selectLinksForExport(@Param("projectUuid") String projectUuid); + + List selectNewCodePeriodsForExport(@Param("projectUuid") String projectUuid); +} diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml new file mode 100644 index 00000000000..6cc5b1566bb --- /dev/null +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/projectdump/ExportSubmitterImpl.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/projectdump/ExportSubmitterImpl.java index 2a9ccaf3669..1661e35297c 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/projectdump/ExportSubmitterImpl.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/projectdump/ExportSubmitterImpl.java @@ -26,6 +26,7 @@ import org.sonar.ce.queue.CeTaskSubmit; import org.sonar.ce.task.CeTask; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.ce.CeTaskTypes; import org.sonar.db.component.ComponentDto; import static com.google.common.base.Preconditions.checkArgument; @@ -52,7 +53,7 @@ public class ExportSubmitterImpl implements ExportSubmitter { CeTaskSubmit submit = ceQueue.prepareSubmit() .setComponent(fromDto(project.get())) - .setType("PROJECT_EXPORT") + .setType(CeTaskTypes.PROJECT_EXPORT) .setSubmitterUuid(submitterUuid) .setCharacteristics(emptyMap()) .build(); diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/projectdump/ws/ExportAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/projectdump/ws/ExportAction.java index 8e436658fc1..e0a8e2a6648 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/projectdump/ws/ExportAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/projectdump/ws/ExportAction.java @@ -19,6 +19,7 @@ */ package org.sonar.server.projectdump.ws; +import org.sonar.api.server.ws.Change; import org.sonar.server.ce.projectdump.ExportSubmitter; import org.sonar.api.server.ws.Request; import org.sonar.api.server.ws.Response; @@ -45,13 +46,13 @@ public class ExportAction implements ProjectDumpAction { @Override public void define(WebService.NewController newController) { WebService.NewAction newAction = newController.createAction(ACTION_KEY) - .setDescription("Triggers project dump so that the project can be copied to another SonarQube server " + - "(see " + ProjectDumpWs.CONTROLLER_PATH + "/import ). " + - "Requires the 'Administer' permission. " + - "This feature is provided by the Governance plugin.") + .setDescription("Triggers project dump so that the project can be imported to another SonarQube server " + + "(see " + ProjectDumpWs.CONTROLLER_PATH + "/import, available in Enterprise Edition). " + + "Requires the 'Administer' permission.") .setSince("1.0") .setPost(true) .setHandler(this) + .setChangelog(new Change("9.2", "Moved from Enterprise Edition to Community Edition")) .setResponseExample(getClass().getResource("example-export.json")); newAction.createParam(PARAMETER_PROJECT_KEY) .setRequired(true) diff --git a/server/sonar-webserver-webapi/src/test/resources/org/sonar/server/projectdump/ws/example-export.json b/server/sonar-webserver-webapi/src/main/resources/org/sonar/server/projectdump/ws/example-export.json similarity index 100% rename from server/sonar-webserver-webapi/src/test/resources/org/sonar/server/projectdump/ws/example-export.json rename to server/sonar-webserver-webapi/src/main/resources/org/sonar/server/projectdump/ws/example-export.json diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/projectdump/ws/ExportActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/projectdump/ws/ExportActionTest.java index f9a05f90a1f..5fdd7ece8ee 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/projectdump/ws/ExportActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/projectdump/ws/ExportActionTest.java @@ -27,6 +27,7 @@ import org.sonar.api.resources.Qualifiers; import org.sonar.api.web.UserRole; import org.sonar.ce.task.CeTask; import org.sonar.db.DbTester; +import org.sonar.db.ce.CeTaskTypes; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ResourceTypesRule; import org.sonar.db.user.UserDto; @@ -151,7 +152,7 @@ public class ExportActionTest { private CeTask createResponseExampleTask() { CeTask.Component component = new CeTask.Component(project.uuid(), project.getDbKey(), project.name()); return new CeTask.Builder() - .setType("PROJECT_EXPORT") // TODO replace with the definitive enum + .setType(CeTaskTypes.PROJECT_EXPORT) .setUuid(TASK_ID) .setComponent(component) .setMainComponent(component) diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 3fbc3ffac43..722141c8359 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -43,6 +43,7 @@ import org.sonar.ce.task.projectanalysis.notification.ReportAnalysisFailureNotif import org.sonar.ce.task.projectanalysis.taskprocessor.AuditPurgeTaskProcessor; import org.sonar.ce.task.projectanalysis.taskprocessor.IssueSyncTaskProcessor; import org.sonar.ce.task.projectanalysis.taskprocessor.ReportTaskProcessor; +import org.sonar.ce.task.projectexport.taskprocessor.ProjectExportTaskProcessor; import org.sonar.core.component.DefaultResourceTypes; import org.sonar.core.extension.CoreExtensionsInstaller; import org.sonar.core.platform.ComponentContainer; @@ -531,6 +532,7 @@ public class PlatformLevel4 extends PlatformLevel { ReportTaskProcessor.class, IssueSyncTaskProcessor.class, AuditPurgeTaskProcessor.class, + ProjectExportTaskProcessor.class, // SonarSource editions PlatformEditionProvider.class, -- 2.39.5