From 453dcdb5e93a05e550094194bead446ac2247361 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Thu, 4 Aug 2016 17:14:51 +0200 Subject: [PATCH] SONAR-7903 persist analysis reports in db instead of file system (data/ce/reports). That allows support of clustering. --- .../ComputeEngineContainerImplTest.java | 6 +- .../src/main/java/org/sonar/ce/CeModule.java | 6 +- .../java/org/sonar/ce/queue/CeQueueImpl.java | 21 +--- .../sonar/ce/queue/report/ReportFiles.java | 96 -------------- .../sonar/ce/queue/report/package-info.java | 23 ---- .../org/sonar/server/ce/ws/SubmitAction.java | 2 +- .../server/computation/CeQueueModule.java | 4 +- .../computation/queue/CeQueueCleaner.java | 34 ++--- .../queue/InternalCeQueueImpl.java | 7 +- .../computation/queue}/ReportSubmitter.java | 15 +-- .../ProjectAnalysisTaskModule.java | 6 +- .../queue/CleanReportQueueListener.java | 39 ------ .../projectanalysis/queue/package-info.java | 23 ---- .../step/ExtractReportStep.java | 35 ++++-- .../org/sonar/ce/queue/CeQueueImplTest.java | 11 +- .../java/org/sonar/server/ce/ws/CeWsTest.java | 2 +- .../sonar/server/ce/ws/SubmitActionTest.java | 2 +- .../server/computation/ReportFilesTest.java | 116 ----------------- .../computation/queue/CeQueueCleanerTest.java | 68 +++++----- .../queue/InternalCeQueueImplTest.java | 18 +-- .../queue}/ReportSubmitterTest.java | 25 ++-- .../queue/CleanReportQueueListenerTest.java | 51 -------- .../step/ExtractReportStepTest.java | 80 ++++++------ .../migrate/1302_create_table_ce_task_data.rb | 36 ++++++ .../src/main/java/org/sonar/db/DaoModule.java | 2 + .../src/main/java/org/sonar/db/DbClient.java | 15 ++- .../src/main/java/org/sonar/db/MyBatis.java | 3 +- .../java/org/sonar/db/ce/CeTaskDataDao.java | 110 ++++++++++++++++ .../org/sonar/db/ce/CeTaskDataMapper.java | 11 +- .../org/sonar/db/version/DatabaseVersion.java | 3 +- .../org/sonar/db/ce/CeTaskDataMapper.xml | 18 +++ .../org/sonar/db/version/rows-h2.sql | 1 + .../org/sonar/db/version/schema-h2.ddl | 8 ++ .../test/java/org/sonar/db/DaoModuleTest.java | 2 +- .../src/test/java/org/sonar/db/DbTester.java | 23 ++-- .../org/sonar/db/ce/CeTaskDataDaoTest.java | 117 ++++++++++++++++++ ...ateUsersExternalIdentityWhenEmptyTest.java | 11 +- 37 files changed, 474 insertions(+), 576 deletions(-) delete mode 100644 server/sonar-server/src/main/java/org/sonar/ce/queue/report/ReportFiles.java delete mode 100644 server/sonar-server/src/main/java/org/sonar/ce/queue/report/package-info.java rename server/sonar-server/src/main/java/org/sonar/{ce/queue/report => server/computation/queue}/ReportSubmitter.java (93%) delete mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListener.java delete mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/package-info.java delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/ReportFilesTest.java rename server/sonar-server/src/test/java/org/sonar/{ce/queue/report => server/computation/queue}/ReportSubmitterTest.java (89%) delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListenerTest.java create mode 100644 server/sonar-web/src/main/webapp/WEB-INF/db/migrate/1302_create_table_ce_task_data.rb create mode 100644 sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataDao.java rename server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueListener.java => sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataMapper.java (78%) create mode 100644 sonar-db/src/main/resources/org/sonar/db/ce/CeTaskDataMapper.xml create mode 100644 sonar-db/src/test/java/org/sonar/db/ce/CeTaskDataDaoTest.java diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java index c9a83fa9177..9fc85e9f8a6 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java @@ -89,9 +89,9 @@ public class ComputeEngineContainerImplTest { .hasSize( CONTAINER_ITSELF + 75 // level 4 - + 7 // content of CeModule + + 6 // content of CeModule + 7 // content of CeQueueModule - + 4 // content of ProjectAnalysisTaskModule + + 3 // content of ProjectAnalysisTaskModule + 4 // content of CeTaskProcessorModule ); assertThat(picoContainer.getParent().getComponentAdapters()).hasSize( @@ -105,7 +105,7 @@ public class ComputeEngineContainerImplTest { assertThat(picoContainer.getParent().getParent().getParent().getComponentAdapters()).hasSize( COMPONENTS_IN_LEVEL_1_AT_CONSTRUCTION + 26 // level 1 - + 46 // content of DaoModule + + 47 // content of DaoModule + 2 // content of EsSearchModule + 55 // content of CorePropertyDefinitions + 1 // content of CePropertyDefinitions diff --git a/server/sonar-server/src/main/java/org/sonar/ce/CeModule.java b/server/sonar-server/src/main/java/org/sonar/ce/CeModule.java index 1f4bb4b7f35..09b13d7434c 100644 --- a/server/sonar-server/src/main/java/org/sonar/ce/CeModule.java +++ b/server/sonar-server/src/main/java/org/sonar/ce/CeModule.java @@ -21,10 +21,9 @@ package org.sonar.ce; import org.sonar.ce.log.CeLogging; import org.sonar.ce.queue.CeQueueImpl; -import org.sonar.ce.queue.report.ReportFiles; -import org.sonar.ce.queue.report.ReportSubmitter; import org.sonar.ce.taskprocessor.ReportTaskProcessorDeclaration; import org.sonar.core.platform.Module; +import org.sonar.server.computation.queue.ReportSubmitter; public class CeModule extends Module { @Override @@ -34,8 +33,7 @@ public class CeModule extends Module { // Queue CeQueueImpl.class, ReportSubmitter.class, - ReportFiles.class, - + // Core tasks processors ReportTaskProcessorDeclaration.class); } diff --git a/server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueImpl.java b/server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueImpl.java index 63adafa270c..316f7a65042 100644 --- a/server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueImpl.java @@ -41,6 +41,7 @@ import org.sonar.db.component.ComponentDto; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Predicates.notNull; import static com.google.common.collect.FluentIterable.from; +import static java.util.Collections.singletonList; import static org.sonar.db.component.ComponentDtoFunctions.toUuid; @ComputeEngineSide @@ -48,22 +49,13 @@ public class CeQueueImpl implements CeQueue { private final DbClient dbClient; private final UuidFactory uuidFactory; - private final CeQueueListener[] listeners; // state private AtomicBoolean submitPaused = new AtomicBoolean(false); - /** - * Constructor in case there is no CeQueueListener - */ public CeQueueImpl(DbClient dbClient, UuidFactory uuidFactory) { - this(dbClient, uuidFactory, new CeQueueListener[] {}); - } - - public CeQueueImpl(DbClient dbClient, UuidFactory uuidFactory, CeQueueListener[] listeners) { this.dbClient = dbClient; this.uuidFactory = uuidFactory; - this.listeners = listeners; } @Override @@ -126,7 +118,7 @@ public class CeQueueImpl implements CeQueue { .toSet(); Map componentDtoByUuid = from(dbClient.componentDao() .selectByUuids(dbSession, componentUuids)) - .uniqueIndex(toUuid()); + .uniqueIndex(toUuid()); return from(dtos) .transform(new CeQueueDtoToCeTask(componentDtoByUuid)) @@ -150,10 +142,9 @@ public class CeQueueImpl implements CeQueue { } protected void cancelImpl(DbSession dbSession, CeQueueDto q) { - CeTask task = loadTask(dbSession, q); CeActivityDto activityDto = new CeActivityDto(q); activityDto.setStatus(CeActivityDto.Status.CANCELED); - remove(dbSession, task, q, activityDto); + remove(dbSession, q, activityDto); } @Override @@ -177,13 +168,11 @@ public class CeQueueImpl implements CeQueue { } } - protected void remove(DbSession dbSession, CeTask task, CeQueueDto queueDto, CeActivityDto activityDto) { + protected void remove(DbSession dbSession, CeQueueDto queueDto, CeActivityDto activityDto) { dbClient.ceActivityDao().insert(dbSession, activityDto); dbClient.ceQueueDao().deleteByUuid(dbSession, queueDto.getUuid()); + dbClient.ceTaskDataDao().deleteByUuids(dbSession, singletonList(queueDto.getUuid())); dbSession.commit(); - for (CeQueueListener listener : listeners) { - listener.onRemoved(task, activityDto.getStatus()); - } } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/ce/queue/report/ReportFiles.java b/server/sonar-server/src/main/java/org/sonar/ce/queue/report/ReportFiles.java deleted file mode 100644 index 223a88ced21..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/ce/queue/report/ReportFiles.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.ce.queue.report; - -import java.io.File; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; -import org.sonar.api.config.Settings; -import org.sonar.api.ce.ComputeEngineSide; -import org.sonar.process.ProcessProperties; - -import static java.lang.String.format; - -@ComputeEngineSide -public class ReportFiles { - - private static final String ZIP_EXTENSION = "zip"; - - private final Settings settings; - - public ReportFiles(Settings settings) { - this.settings = settings; - } - - public void save(String taskUuid, InputStream reportInput) { - File file = fileForUuid(taskUuid); - try { - FileUtils.copyInputStreamToFile(reportInput, file); - } catch (Exception e) { - org.sonar.core.util.FileUtils.deleteQuietly(file); - IOUtils.closeQuietly(reportInput); - throw new IllegalStateException(format("Fail to copy report to file: %s", file.getAbsolutePath()), e); - } - } - - public void deleteIfExists(String taskUuid) { - org.sonar.core.util.FileUtils.deleteQuietly(fileForUuid(taskUuid)); - } - - public void deleteAll() { - File dir = reportDir(); - if (dir.exists()) { - try { - org.sonar.core.util.FileUtils.cleanDirectory(dir); - } catch (Exception e) { - throw new IllegalStateException(format("Fail to clean directory: %s", dir.getAbsolutePath()), e); - } - } - } - - private File reportDir() { - return new File(settings.getString(ProcessProperties.PATH_DATA), "ce/reports"); - } - - /** - * The analysis report to be processed. Can't be null - * but may no exist on file system. - */ - public File fileForUuid(String taskUuid) { - return new File(reportDir(), format("%s.%s", taskUuid, ZIP_EXTENSION)); - } - - public List listUuids() { - List uuids = new ArrayList<>(); - File dir = reportDir(); - if (dir.exists()) { - Collection files = FileUtils.listFiles(dir, new String[]{ZIP_EXTENSION}, false); - for (File file : files) { - uuids.add(FilenameUtils.getBaseName(file.getName())); - } - } - return uuids; - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/ce/queue/report/package-info.java b/server/sonar-server/src/main/java/org/sonar/ce/queue/report/package-info.java deleted file mode 100644 index 3c197b8c457..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/ce/queue/report/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.ce.queue.report; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/main/java/org/sonar/server/ce/ws/SubmitAction.java b/server/sonar-server/src/main/java/org/sonar/server/ce/ws/SubmitAction.java index 33ca21f62e2..f2e264c447c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/ce/ws/SubmitAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/ce/ws/SubmitAction.java @@ -25,7 +25,7 @@ import org.sonar.api.server.ws.Request; import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; import org.sonar.ce.queue.CeTask; -import org.sonar.ce.queue.report.ReportSubmitter; +import org.sonar.server.computation.queue.ReportSubmitter; import org.sonar.server.ws.WsUtils; import org.sonarqube.ws.WsCe; diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/CeQueueModule.java b/server/sonar-server/src/main/java/org/sonar/server/computation/CeQueueModule.java index c2a4506ec8b..c34c9be326f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/CeQueueModule.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/CeQueueModule.java @@ -19,7 +19,6 @@ */ package org.sonar.server.computation; -import org.sonar.ce.queue.report.ReportFiles; import org.sonar.core.platform.Module; import org.sonar.server.computation.monitoring.CEQueueStatusImpl; import org.sonar.server.computation.monitoring.CeTasksMBeanImpl; @@ -40,8 +39,7 @@ public class CeQueueModule extends Module { // queue cleaning CeQueueCleaner.class, - ReportFiles.class, - + // init queue state and queue processing CeQueueInitializer.class); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/queue/CeQueueCleaner.java b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/CeQueueCleaner.java index b7e644a8558..21673dddb30 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/queue/CeQueueCleaner.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/CeQueueCleaner.java @@ -19,17 +19,13 @@ */ package org.sonar.server.computation.queue; -import java.util.HashSet; -import java.util.Set; -import org.sonar.api.platform.ServerUpgradeStatus; +import java.util.List; import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.platform.ServerUpgradeStatus; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import org.sonar.ce.queue.report.ReportFiles; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.ce.CeQueueDto; -import org.sonar.db.ce.CeTaskTypes; /** * Cleans-up the Compute Engine queue and resets the JMX counters. @@ -42,13 +38,11 @@ public class CeQueueCleaner { private final DbClient dbClient; private final ServerUpgradeStatus serverUpgradeStatus; - private final ReportFiles reportFiles; private final InternalCeQueue queue; - public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, ReportFiles reportFiles, InternalCeQueue queue) { + public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, InternalCeQueue queue) { this.dbClient = dbClient; this.serverUpgradeStatus = serverUpgradeStatus; - this.reportFiles = reportFiles; this.queue = queue; } @@ -72,21 +66,11 @@ public class CeQueueCleaner { dbClient.ceQueueDao().resetAllToPendingStatus(dbSession); dbSession.commit(); - // verify that the report files are available for the tasks in queue - Set uuidsInQueue = new HashSet<>(); - for (CeQueueDto queueDto : dbClient.ceQueueDao().selectAllInAscOrder(dbSession)) { - uuidsInQueue.add(queueDto.getUuid()); - if (CeTaskTypes.REPORT.equals(queueDto.getTaskType()) && !reportFiles.fileForUuid(queueDto.getUuid()).exists()) { - // the report is not available on file system - queue.cancel(dbSession, queueDto); - } - } - - // clean-up filesystem - for (String uuid : reportFiles.listUuids()) { - if (!uuidsInQueue.contains(uuid)) { - reportFiles.deleteIfExists(uuid); - } - } + // Reports that have been processed are not kept in database yet. + // They are supposed to be systematically dropped. + // Let's clean-up orphans if any. + List uuids = dbClient.ceTaskDataDao().selectUuidsNotInQueue(dbSession); + dbClient.ceTaskDataDao().deleteByUuids(dbSession, uuids); + dbSession.commit(); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/queue/InternalCeQueueImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/InternalCeQueueImpl.java index 6eb9497d81c..34cd6d4f337 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/queue/InternalCeQueueImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/InternalCeQueueImpl.java @@ -26,7 +26,6 @@ import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.utils.System2; import org.sonar.ce.monitoring.CEQueueStatus; import org.sonar.ce.queue.CeQueueImpl; -import org.sonar.ce.queue.CeQueueListener; import org.sonar.ce.queue.CeTask; import org.sonar.ce.queue.CeTaskResult; import org.sonar.core.util.UuidFactory; @@ -48,8 +47,8 @@ public class InternalCeQueueImpl extends CeQueueImpl implements InternalCeQueue private AtomicBoolean peekPaused = new AtomicBoolean(false); public InternalCeQueueImpl(System2 system2, DbClient dbClient, UuidFactory uuidFactory, - CEQueueStatus queueStatus, CeQueueListener[] listeners) { - super(dbClient, uuidFactory, listeners); + CEQueueStatus queueStatus) { + super(dbClient, uuidFactory); this.system2 = system2; this.dbClient = dbClient; this.queueStatus = queueStatus; @@ -92,7 +91,7 @@ public class InternalCeQueueImpl extends CeQueueImpl implements InternalCeQueue activityDto.setStatus(status); updateQueueStatus(status, activityDto); updateTaskResult(activityDto, taskResult); - remove(dbSession, task, queueDto.get(), activityDto); + remove(dbSession, queueDto.get(), activityDto); } finally { dbClient.closeSession(dbSession); diff --git a/server/sonar-server/src/main/java/org/sonar/ce/queue/report/ReportSubmitter.java b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/ReportSubmitter.java similarity index 93% rename from server/sonar-server/src/main/java/org/sonar/ce/queue/report/ReportSubmitter.java rename to server/sonar-server/src/main/java/org/sonar/server/computation/queue/ReportSubmitter.java index de515bdec77..f60f66626ab 100644 --- a/server/sonar-server/src/main/java/org/sonar/ce/queue/report/ReportSubmitter.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/queue/ReportSubmitter.java @@ -17,14 +17,14 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.ce.queue.report; +package org.sonar.server.computation.queue; import java.io.InputStream; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; -import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.resources.Qualifiers; +import org.sonar.api.server.ServerSide; import org.sonar.ce.queue.CeQueue; import org.sonar.ce.queue.CeTask; import org.sonar.ce.queue.CeTaskSubmit; @@ -41,21 +41,19 @@ import org.sonar.server.user.UserSession; import static org.sonar.core.permission.GlobalPermissions.SCAN_EXECUTION; import static org.sonar.server.user.AbstractUserSession.insufficientPrivilegesException; -@ComputeEngineSide +@ServerSide public class ReportSubmitter { private final CeQueue queue; private final UserSession userSession; - private final ReportFiles reportFiles; private final ComponentService componentService; private final PermissionService permissionService; private final DbClient dbClient; - public ReportSubmitter(CeQueue queue, UserSession userSession, ReportFiles reportFiles, + public ReportSubmitter(CeQueue queue, UserSession userSession, ComponentService componentService, PermissionService permissionService, DbClient dbClient) { this.queue = queue; this.userSession = userSession; - this.reportFiles = reportFiles; this.componentService = componentService; this.permissionService = permissionService; this.dbClient = dbClient; @@ -98,7 +96,10 @@ public class ReportSubmitter { private CeTask submitReport(InputStream reportInput, ComponentDto project) { // the report file must be saved before submitting the task CeTaskSubmit.Builder submit = queue.prepareSubmit(); - reportFiles.save(submit.getUuid(), reportInput); + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.ceTaskDataDao().insert(dbSession, submit.getUuid(), reportInput); + dbSession.commit(); + } submit.setType(CeTaskTypes.REPORT); submit.setComponentUuid(project.uuid()); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/ProjectAnalysisTaskModule.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/ProjectAnalysisTaskModule.java index 7c4bb3b4980..846bb413a2f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/ProjectAnalysisTaskModule.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/ProjectAnalysisTaskModule.java @@ -19,10 +19,9 @@ */ package org.sonar.server.computation.task.projectanalysis; -import org.sonar.ce.queue.report.ReportSubmitter; +import org.sonar.server.computation.queue.ReportSubmitter; import org.sonar.core.platform.Module; import org.sonar.server.computation.task.projectanalysis.container.ContainerFactoryImpl; -import org.sonar.server.computation.task.projectanalysis.queue.CleanReportQueueListener; import org.sonar.server.computation.task.projectanalysis.taskprocessor.ReportTaskProcessor; import org.sonar.server.computation.task.step.ComputationStepExecutor; @@ -30,9 +29,6 @@ public class ProjectAnalysisTaskModule extends Module { @Override protected void configureModule() { add( - // queue - CleanReportQueueListener.class, - // task ContainerFactoryImpl.class, ComputationStepExecutor.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListener.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListener.java deleted file mode 100644 index 8c90ead304e..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListener.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.computation.task.projectanalysis.queue; - -import org.sonar.ce.queue.report.ReportFiles; -import org.sonar.db.ce.CeActivityDto; -import org.sonar.ce.queue.CeQueueListener; -import org.sonar.ce.queue.CeTask; - -public class CleanReportQueueListener implements CeQueueListener { - - private final ReportFiles reportFiles; - - public CleanReportQueueListener(ReportFiles reportFiles) { - this.reportFiles = reportFiles; - } - - @Override - public void onRemoved(CeTask task, CeActivityDto.Status status) { - reportFiles.deleteIfExists(task.getUuid()); - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/package-info.java deleted file mode 100644 index 76cc120f6e1..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/queue/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.server.computation.task.projectanalysis.queue; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java index 566e3afeca4..6d94a12a777 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java @@ -21,13 +21,16 @@ package org.sonar.server.computation.task.projectanalysis.step; import java.io.File; import java.io.IOException; -import org.apache.commons.io.FileUtils; +import java.util.Optional; +import org.sonar.api.utils.MessageException; import org.sonar.api.utils.TempFolder; import org.sonar.api.utils.ZipUtils; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.ce.queue.CeTask; -import org.sonar.ce.queue.report.ReportFiles; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.ce.CeTaskDataDao; import org.sonar.server.computation.task.projectanalysis.batch.MutableBatchReportDirectoryHolder; import org.sonar.server.computation.task.step.ComputationStep; @@ -38,14 +41,14 @@ import org.sonar.server.computation.task.step.ComputationStep; public class ExtractReportStep implements ComputationStep { private static final Logger LOG = Loggers.get(ExtractReportStep.class); - private final ReportFiles reportFiles; + private final DbClient dbClient; private final CeTask task; private final TempFolder tempFolder; private final MutableBatchReportDirectoryHolder reportDirectoryHolder; - public ExtractReportStep(ReportFiles reportFiles, CeTask task, TempFolder tempFolder, + public ExtractReportStep(DbClient dbClient, CeTask task, TempFolder tempFolder, MutableBatchReportDirectoryHolder reportDirectoryHolder) { - this.reportFiles = reportFiles; + this.dbClient = dbClient; this.task = task; this.tempFolder = tempFolder; this.reportDirectoryHolder = reportDirectoryHolder; @@ -53,14 +56,20 @@ public class ExtractReportStep implements ComputationStep { @Override public void execute() { - File dir = tempFolder.newDir(); - File zip = reportFiles.fileForUuid(task.getUuid()); - try { - ZipUtils.unzip(zip, dir); - reportDirectoryHolder.setDirectory(dir); - LOG.info("Analysis report extracted | compressedSize={}", FileUtils.byteCountToDisplaySize(FileUtils.sizeOf(zip))); - } catch (IOException e) { - throw new IllegalStateException(String.format("Fail to unzip %s into %s", zip, dir), e); + try (DbSession dbSession = dbClient.openSession(false)) { + Optional opt = dbClient.ceTaskDataDao().selectData(dbSession, task.getUuid()); + if (opt.isPresent()) { + File unzippedDir = tempFolder.newDir(); + try (CeTaskDataDao.DataStream reportStream = opt.get()) { + ZipUtils.unzip(reportStream.getInputStream(), unzippedDir); + } catch (IOException e) { + throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e); + } + reportDirectoryHolder.setDirectory(unzippedDir); + LOG.info("Analysis report extracted"); + } else { + throw MessageException.of("Analysis report " + task.getUuid() + " is missing in database"); + } } } diff --git a/server/sonar-server/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java b/server/sonar-server/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java index cc75f377a9d..4debf37fc32 100644 --- a/server/sonar-server/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java @@ -40,9 +40,6 @@ import org.sonar.db.component.ComponentTesting; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.startsWith; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; public class CeQueueImplTest { @@ -56,8 +53,7 @@ public class CeQueueImplTest { DbSession session = dbTester.getSession(); UuidFactory uuidFactory = UuidFactoryImpl.INSTANCE; - CeQueueListener listener = mock(CeQueueListener.class); - CeQueue underTest = new CeQueueImpl(dbTester.getDbClient(), uuidFactory, new CeQueueListener[] {listener}); + CeQueue underTest = new CeQueueImpl(dbTester.getDbClient(), uuidFactory); @Test public void submit_returns_task_populated_from_CeTaskSubmit_and_creates_CeQueue_row() { @@ -131,14 +127,12 @@ public class CeQueueImplTest { // ignore boolean canceled = underTest.cancel("UNKNOWN"); assertThat(canceled).isFalse(); - verifyZeroInteractions(listener); canceled = underTest.cancel(task.getUuid()); assertThat(canceled).isTrue(); Optional activity = dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), task.getUuid()); assertThat(activity.isPresent()).isTrue(); assertThat(activity.get().getStatus()).isEqualTo(CeActivityDto.Status.CANCELED); - verify(listener).onRemoved(task, CeActivityDto.Status.CANCELED); } @Test @@ -170,9 +164,6 @@ public class CeQueueImplTest { assertThat(history.get().getStatus()).isEqualTo(CeActivityDto.Status.CANCELED); history = dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), inProgressTask.getUuid()); assertThat(history.isPresent()).isFalse(); - - verify(listener).onRemoved(pendingTask1, CeActivityDto.Status.CANCELED); - verify(listener).onRemoved(pendingTask2, CeActivityDto.Status.CANCELED); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CeWsTest.java b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CeWsTest.java index 1ec0618dbb5..16f7dd9b2f5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CeWsTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CeWsTest.java @@ -22,7 +22,7 @@ package org.sonar.server.ce.ws; import org.junit.Test; import org.mockito.Mockito; import org.sonar.api.server.ws.WebService; -import org.sonar.ce.queue.report.ReportSubmitter; +import org.sonar.server.computation.queue.ReportSubmitter; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; diff --git a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java index b34cb071f23..b7899753a28 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java @@ -25,7 +25,7 @@ import org.mockito.Matchers; import org.sonar.core.util.Protobuf; import org.sonar.db.ce.CeTaskTypes; import org.sonar.ce.queue.CeTask; -import org.sonar.ce.queue.report.ReportSubmitter; +import org.sonar.server.computation.queue.ReportSubmitter; import org.sonarqube.ws.MediaTypes; import org.sonar.server.ws.TestResponse; import org.sonar.server.ws.WsActionTester; diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/ReportFilesTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/ReportFilesTest.java deleted file mode 100644 index f4aee714587..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/ReportFilesTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.computation; - -import java.io.File; -import java.io.IOException; -import org.apache.commons.io.FileUtils; -import org.h2.util.IOUtils; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.sonar.api.config.Settings; -import org.sonar.process.ProcessProperties; -import org.sonar.ce.queue.report.ReportFiles; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ReportFilesTest { - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - File reportDir; - Settings settings = new Settings(); - ReportFiles underTest = new ReportFiles(settings); - - @Before - public void setUp() throws IOException { - File dataDir = temp.newFolder(); - reportDir = new File(dataDir, "ce/reports"); - settings.setProperty(ProcessProperties.PATH_DATA, dataDir.getCanonicalPath()); - } - - @Test - public void save_report() throws IOException { - underTest.save("TASK_1", IOUtils.getInputStreamFromString("{binary}")); - - assertThat(FileUtils.readFileToString(new File(reportDir, "TASK_1.zip"))).isEqualTo("{binary}"); - - } - - @Test - public void deleteIfExists_uuid_does_not_exist() { - // do not fail, does nothing - underTest.deleteIfExists("TASK_1"); - } - - @Test - public void deleteIfExists() throws IOException { - File report = new File(reportDir, "TASK_1.zip"); - FileUtils.touch(report); - assertThat(report).exists(); - - underTest.deleteIfExists("TASK_1"); - assertThat(report).doesNotExist(); - } - - /** - * List the zip files contained in the report directory - */ - @Test - public void listUuids() throws IOException { - FileUtils.touch(new File(reportDir, "TASK_1.zip")); - FileUtils.touch(new File(reportDir, "TASK_2.zip")); - FileUtils.touch(new File(reportDir, "something.else")); - - assertThat(underTest.listUuids()).containsOnly("TASK_1", "TASK_2"); - } - - @Test - public void listUuids_dir_does_not_exist_yet() throws IOException { - FileUtils.deleteQuietly(reportDir); - - assertThat(underTest.listUuids()).isEmpty(); - } - - @Test - public void deleteAll() throws IOException { - FileUtils.touch(new File(reportDir, "TASK_1.zip")); - FileUtils.touch(new File(reportDir, "TASK_2.zip")); - FileUtils.touch(new File(reportDir, "something.else")); - - underTest.deleteAll(); - - // directory still exists but is empty - assertThat(reportDir).exists().isDirectory(); - assertThat(reportDir.listFiles()).isEmpty(); - } - - @Test - public void deleteAll_dir_does_not_exist_yet() throws IOException { - FileUtils.deleteQuietly(reportDir); - - underTest.deleteAll(); - - assertThat(reportDir).doesNotExist(); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/queue/CeQueueCleanerTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/queue/CeQueueCleanerTest.java index 7ea418c5748..923a684213c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/queue/CeQueueCleanerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/queue/CeQueueCleanerTest.java @@ -19,24 +19,20 @@ */ package org.sonar.server.computation.queue; -import java.io.File; import java.io.IOException; +import java.util.Optional; +import org.apache.commons.io.IOUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; import org.sonar.api.platform.ServerUpgradeStatus; import org.sonar.api.utils.System2; -import org.sonar.ce.queue.report.ReportFiles; -import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeTaskDataDao; import org.sonar.db.ce.CeTaskTypes; -import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -49,10 +45,9 @@ public class CeQueueCleanerTest { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - ServerUpgradeStatus serverUpgradeStatus = mock(ServerUpgradeStatus.class); - ReportFiles reportFiles = mock(ReportFiles.class, Mockito.RETURNS_DEEP_STUBS); - InternalCeQueue queue = mock(InternalCeQueue.class); - CeQueueCleaner underTest = new CeQueueCleaner(dbTester.getDbClient(), serverUpgradeStatus, reportFiles, queue); + private ServerUpgradeStatus serverUpgradeStatus = mock(ServerUpgradeStatus.class); + private InternalCeQueue queue = mock(InternalCeQueue.class); + private CeQueueCleaner underTest = new CeQueueCleaner(dbTester.getDbClient(), serverUpgradeStatus, queue); @Test public void reset_in_progress_tasks_to_pending() throws IOException { @@ -74,44 +69,37 @@ public class CeQueueCleanerTest { verify(queue).clear(); } - @Test - public void cancel_task_if_report_file_is_missing() throws IOException { - CeQueueDto task = insertInQueue("TASK_1", CeQueueDto.Status.PENDING, false); - - underTest.clean(dbTester.getSession()); - - verify(queue).cancel(any(DbSession.class), eq(task)); - } - @Test public void delete_orphan_report_files() throws Exception { - // two files on disk but on task in queue - insertInQueue("TASK_1", CeQueueDto.Status.PENDING, true); - when(reportFiles.listUuids()).thenReturn(asList("TASK_1", "TASK_2")); + // analysis reports are persisted but the associated + // task is not in the queue + insertInQueue("TASK_1", CeQueueDto.Status.PENDING); + insertTaskData("TASK_1"); + insertTaskData("TASK_2"); underTest.clean(dbTester.getSession()); - verify(reportFiles).deleteIfExists("TASK_2"); - } + CeTaskDataDao dataDao = dbTester.getDbClient().ceTaskDataDao(); + Optional task1Data = dataDao.selectData(dbTester.getSession(), "TASK_1"); + assertThat(task1Data).isPresent(); + task1Data.get().close(); - private void insertInQueue(String taskUuid, CeQueueDto.Status status) throws IOException { - insertInQueue(taskUuid, status, true); + assertThat(dataDao.selectData(dbTester.getSession(), "TASK_2")).isNotPresent(); } - private CeQueueDto insertInQueue(String taskUuid, CeQueueDto.Status status, boolean createFile) throws IOException { - CeQueueDto queueDto = new CeQueueDto(); - queueDto.setTaskType(CeTaskTypes.REPORT); - queueDto.setComponentUuid("PROJECT_1"); - queueDto.setUuid(taskUuid); - queueDto.setStatus(status); - dbTester.getDbClient().ceQueueDao().insert(dbTester.getSession(), queueDto); + private CeQueueDto insertInQueue(String taskUuid, CeQueueDto.Status status) throws IOException { + CeQueueDto dto = new CeQueueDto(); + dto.setTaskType(CeTaskTypes.REPORT); + dto.setComponentUuid("PROJECT_1"); + dto.setUuid(taskUuid); + dto.setStatus(status); + dbTester.getDbClient().ceQueueDao().insert(dbTester.getSession(), dto); dbTester.getSession().commit(); + return dto; + } - File file = tempFolder.newFile(); - when(reportFiles.fileForUuid(taskUuid)).thenReturn(file); - if (!createFile) { - file.delete(); - } - return queueDto; + private void insertTaskData(String taskUuid) throws IOException { + dbTester.getDbClient().ceTaskDataDao().insert(dbTester.getSession(), taskUuid, IOUtils.toInputStream("{binary}")); + dbTester.getSession().commit(); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/queue/InternalCeQueueImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/queue/InternalCeQueueImplTest.java index 0512c1097e1..2ec21d16240 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/queue/InternalCeQueueImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/queue/InternalCeQueueImplTest.java @@ -28,7 +28,6 @@ import org.junit.rules.ExpectedException; import org.sonar.api.utils.System2; import org.sonar.api.utils.internal.TestSystem2; import org.sonar.ce.monitoring.CEQueueStatus; -import org.sonar.ce.queue.CeQueueListener; import org.sonar.ce.queue.CeTask; import org.sonar.ce.queue.CeTaskResult; import org.sonar.ce.queue.CeTaskSubmit; @@ -46,12 +45,7 @@ import org.sonar.server.computation.monitoring.CEQueueStatusImpl; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.startsWith; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; public class InternalCeQueueImplTest { @@ -69,8 +63,7 @@ public class InternalCeQueueImplTest { UuidFactory uuidFactory = UuidFactoryImpl.INSTANCE; CEQueueStatus queueStatus = new CEQueueStatusImpl(dbTester.getDbClient()); - CeQueueListener listener = mock(CeQueueListener.class); - InternalCeQueue underTest = new InternalCeQueueImpl(system2, dbTester.getDbClient(), uuidFactory, queueStatus, new CeQueueListener[] {listener}); + InternalCeQueue underTest = new InternalCeQueueImpl(system2, dbTester.getDbClient(), uuidFactory, queueStatus); @Test public void submit_returns_task_populated_from_CeTaskSubmit_and_creates_CeQueue_row() { @@ -153,8 +146,6 @@ public class InternalCeQueueImplTest { assertThat(history.get().getStatus()).isEqualTo(CeActivityDto.Status.SUCCESS); assertThat(history.get().getIsLast()).isTrue(); assertThat(history.get().getAnalysisUuid()).isNull(); - - verify(listener).onRemoved(task, CeActivityDto.Status.SUCCESS); } @Test @@ -205,8 +196,6 @@ public class InternalCeQueueImplTest { // no more pending tasks peek = underTest.peek(); assertThat(peek.isPresent()).isFalse(); - - verify(listener, never()).onRemoved(eq(task), any(CeActivityDto.Status.class)); } @Test @@ -225,14 +214,12 @@ public class InternalCeQueueImplTest { // ignore boolean canceled = underTest.cancel("UNKNOWN"); assertThat(canceled).isFalse(); - verifyZeroInteractions(listener); canceled = underTest.cancel(task.getUuid()); assertThat(canceled).isTrue(); Optional activity = dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), task.getUuid()); assertThat(activity.isPresent()).isTrue(); assertThat(activity.get().getStatus()).isEqualTo(CeActivityDto.Status.CANCELED); - verify(listener).onRemoved(task, CeActivityDto.Status.CANCELED); } @Test @@ -262,9 +249,6 @@ public class InternalCeQueueImplTest { assertThat(history.get().getStatus()).isEqualTo(CeActivityDto.Status.CANCELED); history = dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), inProgressTask.getUuid()); assertThat(history.isPresent()).isFalse(); - - verify(listener).onRemoved(pendingTask1, CeActivityDto.Status.CANCELED); - verify(listener).onRemoved(pendingTask2, CeActivityDto.Status.CANCELED); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/ce/queue/report/ReportSubmitterTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/queue/ReportSubmitterTest.java similarity index 89% rename from server/sonar-server/src/test/java/org/sonar/ce/queue/report/ReportSubmitterTest.java rename to server/sonar-server/src/test/java/org/sonar/server/computation/queue/ReportSubmitterTest.java index c98b751ae2f..0e6443b8ce3 100644 --- a/server/sonar-server/src/test/java/org/sonar/ce/queue/report/ReportSubmitterTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/queue/ReportSubmitterTest.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.ce.queue.report; +package org.sonar.server.computation.queue; import org.apache.commons.io.IOUtils; import org.hamcrest.Description; @@ -26,12 +26,13 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; import org.sonar.ce.queue.CeQueue; import org.sonar.ce.queue.CeQueueImpl; import org.sonar.ce.queue.CeTaskSubmit; import org.sonar.core.permission.GlobalPermissions; -import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.DbTester; import org.sonar.db.ce.CeTaskTypes; import org.sonar.db.component.ComponentDto; import org.sonar.server.component.ComponentService; @@ -40,6 +41,7 @@ import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.permission.PermissionService; import org.sonar.server.tester.UserSessionRule; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.argThat; @@ -64,12 +66,13 @@ public class ReportSubmitterTest { @Rule public UserSessionRule userSession = UserSessionRule.standalone(); - CeQueue queue = mock(CeQueueImpl.class); - ReportFiles reportFiles = mock(ReportFiles.class); - ComponentService componentService = mock(ComponentService.class); - PermissionService permissionService = mock(PermissionService.class); - DbClient dbClient = mock(DbClient.class); - ReportSubmitter underTest = new ReportSubmitter(queue, userSession, reportFiles, componentService, permissionService, dbClient); + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private CeQueue queue = mock(CeQueueImpl.class); + private ComponentService componentService = mock(ComponentService.class); + private PermissionService permissionService = mock(PermissionService.class); + private ReportSubmitter underTest = new ReportSubmitter(queue, userSession, componentService, permissionService, dbTester.getDbClient()); @Test public void submit_a_report_on_existing_project() { @@ -80,6 +83,7 @@ public class ReportSubmitterTest { underTest.submit(PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + verifyReportIsPersisted(TASK_UUID); verifyZeroInteractions(permissionService); verify(queue).submit(argThat(new TypeSafeMatcher() { @Override @@ -107,6 +111,7 @@ public class ReportSubmitterTest { underTest.submit(PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + verifyReportIsPersisted(TASK_UUID); verify(permissionService).applyDefaultPermissionTemplate(any(DbSession.class), eq(PROJECT_KEY)); verify(queue).submit(argThat(new TypeSafeMatcher() { @Override @@ -181,4 +186,8 @@ public class ReportSubmitterTest { underTest.submit(PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); } + private void verifyReportIsPersisted(String taskUuid) { + assertThat(dbTester.selectFirst("select task_uuid from ce_task_data where task_uuid='" + taskUuid + "'")).isNotNull(); + } + } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListenerTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListenerTest.java deleted file mode 100644 index 3d3c2bfe560..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/queue/CleanReportQueueListenerTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.computation.task.projectanalysis.queue; - -import org.junit.Test; -import org.sonar.ce.queue.CeTask; -import org.sonar.ce.queue.report.ReportFiles; -import org.sonar.db.ce.CeActivityDto; -import org.sonar.db.ce.CeTaskTypes; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -public class CleanReportQueueListenerTest { - - ReportFiles reportFiles = mock(ReportFiles.class); - CleanReportQueueListener underTest = new CleanReportQueueListener(reportFiles); - - @Test - public void remove_report_file_if_success() { - CeTask task = new CeTask.Builder().setUuid("TASK_1").setType(CeTaskTypes.REPORT).setComponentUuid("PROJECT_1").setSubmitterLogin(null).build(); - - underTest.onRemoved(task, CeActivityDto.Status.SUCCESS); - verify(reportFiles).deleteIfExists("TASK_1"); - } - - @Test - public void remove_report_file_if_failure() { - CeTask task = new CeTask.Builder().setUuid("TASK_1").setType(CeTaskTypes.REPORT).setComponentUuid("PROJECT_1").setSubmitterLogin(null).build(); - - underTest.onRemoved(task, CeActivityDto.Status.FAILED); - verify(reportFiles).deleteIfExists("TASK_1"); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStepTest.java index eb6c586d1e9..f6077caf95c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStepTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStepTest.java @@ -21,29 +21,29 @@ package org.sonar.server.computation.task.projectanalysis.step; import java.io.File; import java.io.IOException; +import java.io.InputStream; import org.apache.commons.io.FileUtils; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.sonar.api.utils.MessageException; +import org.sonar.api.utils.System2; import org.sonar.api.utils.ZipUtils; import org.sonar.api.utils.internal.JUnitTempFolder; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.queue.CeTask; +import org.sonar.db.DbTester; import org.sonar.db.ce.CeTaskTypes; +import org.sonar.server.computation.task.projectanalysis.batch.BatchReportDirectoryHolderImpl; import org.sonar.server.computation.task.projectanalysis.batch.MutableBatchReportDirectoryHolder; -import org.sonar.ce.queue.CeTask; -import org.sonar.ce.queue.report.ReportFiles; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.assertj.core.api.Assertions.assertThat; public class ExtractReportStepTest { - public static final String TASK_UUID = "1"; + private static final String TASK_UUID = "1"; + @Rule public JUnitTempFolder tempFolder = new JUnitTempFolder(); @@ -53,52 +53,46 @@ public class ExtractReportStepTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - MutableBatchReportDirectoryHolder reportDirectoryHolder = mock(MutableBatchReportDirectoryHolder.class); - ReportFiles reportFiles = mock(ReportFiles.class); - CeTask ceTask = new CeTask.Builder().setType(CeTaskTypes.REPORT).setUuid(TASK_UUID).build(); + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private MutableBatchReportDirectoryHolder reportDirectoryHolder = new BatchReportDirectoryHolderImpl(); + private CeTask ceTask = new CeTask.Builder().setType(CeTaskTypes.REPORT).setUuid(TASK_UUID).build(); - ExtractReportStep underTest = new ExtractReportStep(reportFiles, ceTask, tempFolder, reportDirectoryHolder); + private ExtractReportStep underTest = new ExtractReportStep(dbTester.getDbClient(), ceTask, tempFolder, reportDirectoryHolder); @Test public void fail_if_report_zip_does_not_exist() throws Exception { - File zip = tempFolder.newFile(); - FileUtils.forceDelete(zip); - expectedException.expect(IllegalStateException.class); - expectedException.expectMessage("Fail to unzip " + zip.getPath()); - - when(reportFiles.fileForUuid(TASK_UUID)).thenReturn(zip); + expectedException.expect(MessageException.class); + expectedException.expectMessage("Analysis report 1 is missing in database"); underTest.execute(); } @Test public void unzip_report() throws Exception { - File zipDir = tempFolder.newDir(); - final File metadataFile = new File(zipDir, "metadata.pb"); - FileUtils.write(metadataFile, "{report}"); - File zip = tempFolder.newFile(); - ZipUtils.zipDir(zipDir, zip); - when(reportFiles.fileForUuid(TASK_UUID)).thenReturn(zip); + File reportFile = generateReport(); + try (InputStream input = FileUtils.openInputStream(reportFile)) { + dbTester.getDbClient().ceTaskDataDao().insert(dbTester.getSession(), TASK_UUID, input); + } + dbTester.getSession().commit(); + dbTester.getSession().close(); underTest.execute(); - verify(reportDirectoryHolder).setDirectory(argThat(new TypeSafeMatcher() { - @Override - protected boolean matchesSafely(File dir) { - try { - return dir.isDirectory() && dir.exists() && - // directory contains the uncompressed report (which contains only metadata.pb in this test) - dir.listFiles().length == 1 && - FileUtils.contentEquals(dir.listFiles()[0], metadataFile); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - @Override - public void describeTo(Description description) { - - } - })); + // directory contains the uncompressed report (which contains only metadata.pb in this test) + File unzippedDir = reportDirectoryHolder.getDirectory(); + assertThat(unzippedDir).isDirectory().exists(); + assertThat(unzippedDir.listFiles()).hasSize(1); + assertThat(new File(unzippedDir, "metadata.pb")).hasContent("{metadata}"); + } + + private File generateReport() throws IOException { + File zipDir = tempFolder.newDir(); + File metadataFile = new File(zipDir, "metadata.pb"); + FileUtils.write(metadataFile, "{metadata}"); + File zip = tempFolder.newFile(); + ZipUtils.zipDir(zipDir, zip); + return zip; } } diff --git a/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/1302_create_table_ce_task_data.rb b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/1302_create_table_ce_task_data.rb new file mode 100644 index 00000000000..c46371778b3 --- /dev/null +++ b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/1302_create_table_ce_task_data.rb @@ -0,0 +1,36 @@ +# +# SonarQube, open source software quality management tool. +# Copyright (C) 2008-2014 SonarSource +# mailto:contact AT sonarsource DOT com +# +# SonarQube is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# SonarQube is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# +# SonarQube 6.1 +# +class CreateTableCeTaskData < ActiveRecord::Migration + + def self.up + # FIXME define primary key + create_table 'ce_task_data', :id => false do |t| + t.column 'task_uuid', :string, :limit => 40, :null => false + t.column 'data', :binary, :null => true + t.column 'created_at', :big_integer, :null => false + t.column 'updated_at', :big_integer, :null => false + end + add_index 'ce_task_data', 'task_uuid', :name => 'ce_task_data_uuid', :unique => true + end +end diff --git a/sonar-db/src/main/java/org/sonar/db/DaoModule.java b/sonar-db/src/main/java/org/sonar/db/DaoModule.java index 4649f70ecc2..1f73c55a4c9 100644 --- a/sonar-db/src/main/java/org/sonar/db/DaoModule.java +++ b/sonar-db/src/main/java/org/sonar/db/DaoModule.java @@ -25,6 +25,7 @@ import org.sonar.core.platform.Module; import org.sonar.db.activity.ActivityDao; import org.sonar.db.ce.CeActivityDao; import org.sonar.db.ce.CeQueueDao; +import org.sonar.db.ce.CeTaskDataDao; import org.sonar.db.component.ComponentDao; import org.sonar.db.component.ComponentLinkDao; import org.sonar.db.component.ResourceDao; @@ -77,6 +78,7 @@ public class DaoModule extends Module { AuthorizationDao.class, CeActivityDao.class, CeQueueDao.class, + CeTaskDataDao.class, ComponentDao.class, ComponentLinkDao.class, CustomMeasureDao.class, diff --git a/sonar-db/src/main/java/org/sonar/db/DbClient.java b/sonar-db/src/main/java/org/sonar/db/DbClient.java index e883d1cb62b..5564c6c145b 100644 --- a/sonar-db/src/main/java/org/sonar/db/DbClient.java +++ b/sonar-db/src/main/java/org/sonar/db/DbClient.java @@ -25,6 +25,7 @@ import javax.annotation.Nullable; import org.sonar.db.activity.ActivityDao; import org.sonar.db.ce.CeActivityDao; import org.sonar.db.ce.CeQueueDao; +import org.sonar.db.ce.CeTaskDataDao; import org.sonar.db.component.ComponentDao; import org.sonar.db.component.ComponentLinkDao; import org.sonar.db.component.ResourceDao; @@ -97,8 +98,9 @@ public class DbClient { private final IssueFilterDao issueFilterDao; private final IssueFilterFavouriteDao issueFilterFavouriteDao; private final IssueChangeDao issueChangeDao; - private final CeQueueDao ceQueueDao; private final CeActivityDao ceActivityDao; + private final CeQueueDao ceQueueDao; + private final CeTaskDataDao ceTaskDataDao; private final DashboardDao dashboardDao; private final ActiveDashboardDao activeDashboardDao; private final WidgetDao widgetDao; @@ -152,8 +154,9 @@ public class DbClient { issueFilterDao = getDao(map, IssueFilterDao.class); issueFilterFavouriteDao = getDao(map, IssueFilterFavouriteDao.class); issueChangeDao = getDao(map, IssueChangeDao.class); - ceQueueDao = getDao(map, CeQueueDao.class); ceActivityDao = getDao(map, CeActivityDao.class); + ceQueueDao = getDao(map, CeQueueDao.class); + ceTaskDataDao = getDao(map, CeTaskDataDao.class); dashboardDao = getDao(map, DashboardDao.class); activeDashboardDao = getDao(map, ActiveDashboardDao.class); widgetDao = getDao(map, WidgetDao.class); @@ -290,12 +293,16 @@ public class DbClient { return permissionTemplateCharacteristicDao; } + public CeActivityDao ceActivityDao() { + return ceActivityDao; + } + public CeQueueDao ceQueueDao() { return ceQueueDao; } - public CeActivityDao ceActivityDao() { - return ceActivityDao; + public CeTaskDataDao ceTaskDataDao() { + return ceTaskDataDao; } public DashboardDao dashboardDao() { diff --git a/sonar-db/src/main/java/org/sonar/db/MyBatis.java b/sonar-db/src/main/java/org/sonar/db/MyBatis.java index 45a675457b3..4e974a06572 100644 --- a/sonar-db/src/main/java/org/sonar/db/MyBatis.java +++ b/sonar-db/src/main/java/org/sonar/db/MyBatis.java @@ -33,6 +33,7 @@ import org.sonar.db.activity.ActivityDto; import org.sonar.db.activity.ActivityMapper; import org.sonar.db.ce.CeActivityMapper; import org.sonar.db.ce.CeQueueMapper; +import org.sonar.db.ce.CeTaskDataMapper; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentDtoWithSnapshotId; import org.sonar.db.component.ComponentLinkDto; @@ -233,7 +234,7 @@ public class MyBatis { GroupMembershipMapper.class, QualityProfileMapper.class, ActiveRuleMapper.class, MeasureMapper.class, MetricMapper.class, CustomMeasureMapper.class, QualityGateMapper.class, QualityGateConditionMapper.class, ComponentMapper.class, SnapshotMapper.class, ProjectQgateAssociationMapper.class, EventMapper.class, - CeQueueMapper.class, CeActivityMapper.class, ComponentLinkMapper.class, + CeQueueMapper.class, CeActivityMapper.class, CeTaskDataMapper.class, ComponentLinkMapper.class, Migration45Mapper.class, Migration50Mapper.class, Migration53Mapper.class }; confBuilder.loadMappers(mappers); diff --git a/sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataDao.java b/sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataDao.java new file mode 100644 index 00000000000..36d99c5e1aa --- /dev/null +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataDao.java @@ -0,0 +1,110 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.ce; + +import java.io.InputStream; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Optional; +import org.apache.commons.io.IOUtils; +import org.sonar.api.utils.System2; +import org.sonar.db.Dao; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbSession; + +public class CeTaskDataDao implements Dao { + + private final System2 system; + + public CeTaskDataDao(System2 system) { + this.system = system; + } + + public void insert(DbSession dbSession, String taskUuid, InputStream data) { + long now = system.now(); + try (PreparedStatement stmt = dbSession.getConnection().prepareStatement( + "INSERT INTO ce_task_data (task_uuid, created_at, updated_at, data) VALUES (?, ?, ?, ?)")) { + stmt.setString(1, taskUuid); + stmt.setLong(2, now); + stmt.setLong(3, now); + stmt.setBinaryStream(4, data); + stmt.executeUpdate(); + } catch (SQLException e) { + throw new IllegalStateException("Fail to insert data of CE task " + taskUuid, e); + } + } + + public Optional selectData(DbSession dbSession, String taskUuid) { + PreparedStatement stmt = null; + ResultSet rs = null; + DataStream result = null; + try { + stmt = dbSession.getConnection().prepareStatement("SELECT data FROM ce_task_data WHERE task_uuid=? AND data IS NOT NULL"); + stmt.setString(1, taskUuid); + rs = stmt.executeQuery(); + if (rs.next()) { + result = new DataStream(stmt, rs, rs.getBinaryStream(1)); + return Optional.of(result); + } + return Optional.empty(); + } catch (SQLException e) { + throw new IllegalStateException("Fail to select data of CE task " + taskUuid, e); + } finally { + if (result == null) { + DatabaseUtils.closeQuietly(rs); + DatabaseUtils.closeQuietly(stmt); + } + } + } + + public List selectUuidsNotInQueue(DbSession dbSession) { + return dbSession.getMapper(CeTaskDataMapper.class).selectUuidsNotInQueue(); + } + + public void deleteByUuids(DbSession dbSession, List uuids) { + CeTaskDataMapper mapper = dbSession.getMapper(CeTaskDataMapper.class); + DatabaseUtils.executeLargeUpdates(uuids, mapper::deleteByUuids); + } + + public static class DataStream implements AutoCloseable { + private final PreparedStatement stmt; + private final ResultSet rs; + private final InputStream stream; + + private DataStream(PreparedStatement stmt, ResultSet rs, InputStream stream) { + this.stmt = stmt; + this.rs = rs; + this.stream = stream; + } + + public InputStream getInputStream() { + return stream; + } + + @Override + public void close() { + IOUtils.closeQuietly(stream); + DatabaseUtils.closeQuietly(rs); + DatabaseUtils.closeQuietly(stmt); + } + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueListener.java b/sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataMapper.java similarity index 78% rename from server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueListener.java rename to sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataMapper.java index 90e3a0a0915..728f6d3210c 100644 --- a/server/sonar-server/src/main/java/org/sonar/ce/queue/CeQueueListener.java +++ b/sonar-db/src/main/java/org/sonar/db/ce/CeTaskDataMapper.java @@ -17,12 +17,15 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.ce.queue; +package org.sonar.db.ce; -import org.sonar.db.ce.CeActivityDto; +import java.util.List; +import org.apache.ibatis.annotations.Param; -public interface CeQueueListener { +public interface CeTaskDataMapper { - void onRemoved(CeTask task, CeActivityDto.Status status); + void deleteByUuids(@Param("uuids") List uuids); + + List selectUuidsNotInQueue(); } diff --git a/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java b/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java index 3ddf44e75e6..8be4800dac9 100644 --- a/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java +++ b/sonar-db/src/main/java/org/sonar/db/version/DatabaseVersion.java @@ -30,7 +30,7 @@ import org.sonar.db.MyBatis; public class DatabaseVersion { - public static final int LAST_VERSION = 1_301; + public static final int LAST_VERSION = 1_302; /** * The minimum supported version which can be upgraded. Lower @@ -52,6 +52,7 @@ public class DatabaseVersion { "authors", "ce_activity", "ce_queue", + "ce_task_data", "dashboards", "duplications_index", "events", diff --git a/sonar-db/src/main/resources/org/sonar/db/ce/CeTaskDataMapper.xml b/sonar-db/src/main/resources/org/sonar/db/ce/CeTaskDataMapper.xml new file mode 100644 index 00000000000..ea36ab63069 --- /dev/null +++ b/sonar-db/src/main/resources/org/sonar/db/ce/CeTaskDataMapper.xml @@ -0,0 +1,18 @@ + + + + + + + + + delete from ce_task_data + where task_uuid in #{uuid} + + + diff --git a/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql b/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql index af97f0ae499..02da496f805 100644 --- a/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql +++ b/sonar-db/src/main/resources/org/sonar/db/version/rows-h2.sql @@ -488,6 +488,7 @@ INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('1277'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('1300'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('1301'); +INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('1302'); INSERT INTO USERS(ID, LOGIN, NAME, EMAIL, EXTERNAL_IDENTITY, EXTERNAL_IDENTITY_PROVIDER, USER_LOCAL, CRYPTED_PASSWORD, SALT, CREATED_AT, UPDATED_AT) VALUES (1, 'admin', 'Administrator', '', 'admin', 'sonarqube', true, 'a373a0e667abb2604c1fd571eb4ad47fe8cc0878', '48bc4b0d93179b5103fd3885ea9119498e9d161b', '1418215735482', '1418215735482'); ALTER TABLE USERS ALTER COLUMN ID RESTART WITH 2; diff --git a/sonar-db/src/main/resources/org/sonar/db/version/schema-h2.ddl b/sonar-db/src/main/resources/org/sonar/db/version/schema-h2.ddl index f9704f1a0c6..15fca8d0a79 100644 --- a/sonar-db/src/main/resources/org/sonar/db/version/schema-h2.ddl +++ b/sonar-db/src/main/resources/org/sonar/db/version/schema-h2.ddl @@ -533,6 +533,14 @@ CREATE TABLE "CE_ACTIVITY" ( "EXECUTION_TIME_MS" BIGINT NULL ); +CREATE TABLE "CE_TASK_DATA" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "TASK_UUID" VARCHAR(40) NOT NULL, + "DATA" BLOB(167772150), + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL +); + CREATE TABLE "USER_TOKENS" ( "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), "LOGIN" VARCHAR(255) NOT NULL, diff --git a/sonar-db/src/test/java/org/sonar/db/DaoModuleTest.java b/sonar-db/src/test/java/org/sonar/db/DaoModuleTest.java index c76d29e2dc1..5e8d8910d50 100644 --- a/sonar-db/src/test/java/org/sonar/db/DaoModuleTest.java +++ b/sonar-db/src/test/java/org/sonar/db/DaoModuleTest.java @@ -29,6 +29,6 @@ public class DaoModuleTest { public void verify_count_of_added_components() { ComponentContainer container = new ComponentContainer(); new DaoModule().configure(container); - assertThat(container.size()).isEqualTo(2 + 46); + assertThat(container.size()).isEqualTo(2 + 47); } } diff --git a/sonar-db/src/test/java/org/sonar/db/DbTester.java b/sonar-db/src/test/java/org/sonar/db/DbTester.java index 170a863862f..b729146fbf1 100644 --- a/sonar-db/src/test/java/org/sonar/db/DbTester.java +++ b/sonar-db/src/test/java/org/sonar/db/DbTester.java @@ -21,7 +21,9 @@ package org.sonar.db; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import java.io.InputStream; import java.math.BigDecimal; import java.sql.Clob; @@ -130,7 +132,7 @@ public class DbTester extends ExternalResource { return client; } - public void executeUpdateSql(String sql, String... params) { + public void executeUpdateSql(String sql, Object... params) { try (Connection connection = db.getDatabase().getDataSource().getConnection()) { new QueryRunner().update(connection, sql, params); } catch (Exception e) { @@ -144,15 +146,16 @@ public class DbTester extends ExternalResource { * * @param valuesByColumn column name and value pairs, if any value is null, the associated column won't be inserted */ - public void executeInsert(String table, String... valuesByColumn) { - executeInsert(table, mapOf(valuesByColumn)); + public void executeInsert(String table, String firstColumn, Object... others) { + executeInsert(table, mapOf(firstColumn, others)); } - private static Map mapOf(String... values) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (int i = 0; i < values.length; i++) { - String key = values[i]; - String value = values[i + 1]; + private static Map mapOf(String firstColumn, Object... values) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + List args = Lists.asList(firstColumn, values); + for (int i = 0; i < args.size(); i++) { + String key = args.get(i).toString(); + Object value = args.get(i + 1); if (value != null) { builder.put(key, value); } @@ -165,7 +168,7 @@ public class DbTester extends ExternalResource { * Very simple helper method to insert some data into a table. * It's the responsibility of the caller to convert column values to string. */ - public void executeInsert(String table, Map valuesByColumn) { + public void executeInsert(String table, Map valuesByColumn) { if (valuesByColumn.isEmpty()) { throw new IllegalArgumentException("Values cannot be empty"); } @@ -175,7 +178,7 @@ public class DbTester extends ExternalResource { ") values (" + COMMA_JOINER.join(Collections.nCopies(valuesByColumn.size(), '?')) + ")"; - executeUpdateSql(sql, valuesByColumn.values().toArray(new String[valuesByColumn.size()])); + executeUpdateSql(sql, valuesByColumn.values().toArray(new Object[valuesByColumn.size()])); } /** diff --git a/sonar-db/src/test/java/org/sonar/db/ce/CeTaskDataDaoTest.java b/sonar-db/src/test/java/org/sonar/db/ce/CeTaskDataDaoTest.java new file mode 100644 index 00000000000..bacd7101fce --- /dev/null +++ b/sonar-db/src/test/java/org/sonar/db/ce/CeTaskDataDaoTest.java @@ -0,0 +1,117 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.ce; + +import java.io.InputStream; +import java.util.Optional; +import org.apache.commons.io.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CeTaskDataDaoTest { + + private static final String A_UUID = "U1"; + private static final String SOME_DATA = "this_is_a_report"; + private static final long NOW = 1_500_000_000_000L; + private static final String TABLE_NAME = "CE_TASK_DATA"; + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private System2 system = mock(System2.class); + private CeTaskDataDao underTest = new CeTaskDataDao(system); + + @Test + public void insert_and_select_data_stream() throws Exception { + when(system.now()).thenReturn(NOW); + + InputStream report = IOUtils.toInputStream(SOME_DATA); + underTest.insert(dbTester.getSession(), A_UUID, report); + + Optional result = underTest.selectData(dbTester.getSession(), A_UUID); + assertThat(result).isPresent(); + try { + assertThat(IOUtils.toString(result.get().getInputStream())).isEqualTo(SOME_DATA); + } finally { + result.get().close(); + } + } + + @Test + public void fail_to_insert_invalid_row() throws Exception { + expectedException.expectMessage("Fail to insert data of CE task null"); + underTest.insert(dbTester.getSession(), null, IOUtils.toInputStream(SOME_DATA)); + } + + @Test + public void selectData_returns_absent_if_uuid_not_found() { + Optional result = underTest.selectData(dbTester.getSession(), A_UUID); + assertThat(result).isNotPresent(); + } + + @Test + public void selectData_returns_absent_if_uuid_exists_but_data_is_null() { + insertData(A_UUID); + dbTester.commit(); + + Optional result = underTest.selectData(dbTester.getSession(), A_UUID); + assertThat(result).isNotPresent(); + } + + @Test + public void selectUuidsNotInQueue() { + assertThat(underTest.selectUuidsNotInQueue(dbTester.getSession())).isEmpty(); + + insertData("U1"); + insertData("U2"); + assertThat(underTest.selectUuidsNotInQueue(dbTester.getSession())).containsOnly("U1", "U2"); + + CeQueueDto inQueue = new CeQueueDto().setUuid("U2").setTaskType(CeTaskTypes.REPORT).setStatus(CeQueueDto.Status.IN_PROGRESS); + new CeQueueDao(system).insert(dbTester.getSession(), inQueue); + assertThat(underTest.selectUuidsNotInQueue(dbTester.getSession())).containsOnly("U1"); + } + + @Test + public void deleteByUuids() { + insertData(A_UUID); + assertThat(dbTester.countRowsOfTable(TABLE_NAME)).isEqualTo(1); + + underTest.deleteByUuids(dbTester.getSession(), asList(A_UUID)); + dbTester.commit(); + assertThat(dbTester.countRowsOfTable(TABLE_NAME)).isEqualTo(0); + } + + private void insertData(String uuid) { + + dbTester.executeInsert(TABLE_NAME, "TASK_UUID", uuid, "CREATED_AT", NOW, "UPDATED_AT", NOW); + dbTester.commit(); + } +} diff --git a/sonar-db/src/test/java/org/sonar/db/version/v56/UpdateUsersExternalIdentityWhenEmptyTest.java b/sonar-db/src/test/java/org/sonar/db/version/v56/UpdateUsersExternalIdentityWhenEmptyTest.java index 7f0abdcc7db..60471734237 100644 --- a/sonar-db/src/test/java/org/sonar/db/version/v56/UpdateUsersExternalIdentityWhenEmptyTest.java +++ b/sonar-db/src/test/java/org/sonar/db/version/v56/UpdateUsersExternalIdentityWhenEmptyTest.java @@ -19,7 +19,7 @@ */ package org.sonar.db.version.v56; -import com.google.common.collect.ImmutableMap; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; import org.junit.Before; @@ -29,7 +29,6 @@ import org.sonar.api.utils.System2; import org.sonar.db.DbTester; import org.sonar.db.version.MigrationStep; -import static com.google.common.collect.Maps.newHashMap; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -73,10 +72,10 @@ public class UpdateUsersExternalIdentityWhenEmptyTest { } private void insertUser(String login, @Nullable String externalIdentity, @Nullable String externalIdentityProvider, long updatedAt) { - Map params = newHashMap(ImmutableMap.of( - "LOGIN", login, - "CREATED_AT", Long.toString(PAST), - "UPDATED_AT", Long.toString(updatedAt))); + Map params = new HashMap<>(); + params.put("LOGIN", login); + params.put("CREATED_AT", Long.toString(PAST)); + params.put("UPDATED_AT", Long.toString(updatedAt)); if (externalIdentity != null) { params.put("EXTERNAL_IDENTITY", externalIdentity); } -- 2.39.5