From 9a934db810c4b91b13ff6fafa2bb50dcdd933d63 Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 9 Apr 2015 11:53:32 +0200 Subject: [PATCH] SONAR-6258 Persist source, coverage and scm into file sources --- .../source/ComputeFileSourceData.java | 105 +++++ .../source/CoverageLineReader.java | 101 +++++ .../server/computation/source/LineReader.java | 29 ++ .../computation/source/ScmLineReader.java | 55 +++ .../computation/step/ComputationSteps.java | 3 + .../computation/step/PersistCoverageStep.java | 112 ----- .../step/PersistFileSourcesStep.java | 186 ++++++++ .../source/ComputeFileSourceDataTest.java | 110 +++++ .../source/CoverageLineReaderTest.java | 193 +++++++++ .../source/ReportIteratorTest.java | 10 + .../computation/source/ScmLineReaderTest.java | 129 ++++++ .../step/PersistCoverageStepTest.java | 182 -------- .../step/PersistFileSourcesStepTest.java | 410 ++++++++++++++++++ .../sonar/core/source/db/FileSourceDao.java | 12 +- 14 files changed, 1341 insertions(+), 296 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/source/ComputeFileSourceData.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/source/CoverageLineReader.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/source/LineReader.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/source/ScmLineReader.java delete mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistCoverageStep.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/source/ComputeFileSourceDataTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/source/CoverageLineReaderTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/source/ScmLineReaderTest.java delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistCoverageStepTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/source/ComputeFileSourceData.java b/server/sonar-server/src/main/java/org/sonar/server/computation/source/ComputeFileSourceData.java new file mode 100644 index 00000000000..09e659bfc01 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/source/ComputeFileSourceData.java @@ -0,0 +1,105 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.source; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang.StringUtils; +import org.sonar.server.source.db.FileSourceDb; + +import java.security.MessageDigest; +import java.util.Iterator; +import java.util.List; + +import static com.google.common.base.Charsets.UTF_8; + +public class ComputeFileSourceData { + + private final List lineReaders; + private final Iterator linesIterator; + + private final int numberOfLines; + private int currentLine; + + public ComputeFileSourceData(Iterator sourceLinesIterator, List dataLineReaders, int numberOfLines) { + this.lineReaders = dataLineReaders; + this.linesIterator = sourceLinesIterator; + this.numberOfLines = numberOfLines; + this.currentLine = 0; + } + + public Data compute() { + Data data = new Data(); + while (linesIterator.hasNext()) { + read(data, linesIterator.next()); + } + // Process last line + if (currentLine < numberOfLines) { + read(data, ""); + } + return data; + } + + private void read(Data data, String source) { + FileSourceDb.Line.Builder lineBuilder = data.fileSourceBuilder.addLinesBuilder().setSource(source); + + currentLine++; + String sourceLine = lineBuilder.getSource(); + data.lineHashes.append(computeLineChecksum(sourceLine)).append("\n"); + data.srcMd5Digest.update((sourceLine + "\n").getBytes(UTF_8)); + lineBuilder.setLine(currentLine); + for (LineReader lineReader : lineReaders) { + lineReader.read(lineBuilder); + } + } + + private static String computeLineChecksum(String line) { + String reducedLine = StringUtils.replaceChars(line, "\t ", ""); + if (reducedLine.isEmpty()) { + return ""; + } + return DigestUtils.md5Hex(reducedLine); + } + + public static class Data { + private final StringBuilder lineHashes; + private final MessageDigest srcMd5Digest; + private final FileSourceDb.Data.Builder fileSourceBuilder; + + public Data() { + this.fileSourceBuilder = FileSourceDb.Data.newBuilder(); + this.lineHashes = new StringBuilder(); + this.srcMd5Digest = DigestUtils.getMd5Digest(); + } + + public String getSrcHash() { + return Hex.encodeHexString(srcMd5Digest.digest()); + } + + public String getLineHashes() { + return lineHashes.toString(); + } + + public FileSourceDb.Data getFileSourceData() { + return fileSourceBuilder.build(); + } + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/source/CoverageLineReader.java b/server/sonar-server/src/main/java/org/sonar/server/computation/source/CoverageLineReader.java new file mode 100644 index 00000000000..7f6684a714d --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/source/CoverageLineReader.java @@ -0,0 +1,101 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.source; + +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.server.source.db.FileSourceDb; + +import javax.annotation.CheckForNull; + +import java.util.Iterator; + +public class CoverageLineReader implements LineReader { + + private final Iterator coverageIterator; + private BatchReport.Coverage coverage; + + public CoverageLineReader(Iterator coverageIterator) { + this.coverageIterator = coverageIterator; + } + + @Override + public void read(FileSourceDb.Line.Builder lineBuilder) { + BatchReport.Coverage reportCoverage = getNextLineCoverageIfMatchLine(lineBuilder.getLine()); + if (reportCoverage != null) { + processUnitTest(lineBuilder, reportCoverage); + processIntegrationTest(lineBuilder, reportCoverage); + processOverallTest(lineBuilder, reportCoverage); + coverage = null; + } + } + + private static void processUnitTest(FileSourceDb.Line.Builder lineBuilder, BatchReport.Coverage reportCoverage){ + if (hasUnitTests(reportCoverage)) { + lineBuilder.setUtLineHits(1); + } + if (reportCoverage.hasConditions() && reportCoverage.hasUtCoveredConditions()) { + lineBuilder.setUtConditions(reportCoverage.getConditions()); + lineBuilder.setUtCoveredConditions(reportCoverage.getUtCoveredConditions()); + } + } + + private static boolean hasUnitTests(BatchReport.Coverage reportCoverage){ + return reportCoverage.hasUtHits() && reportCoverage.getUtHits(); + } + + private static void processIntegrationTest(FileSourceDb.Line.Builder lineBuilder, BatchReport.Coverage reportCoverage){ + if (hasIntegrationTests(reportCoverage)) { + lineBuilder.setItLineHits(1); + } + if (reportCoverage.hasConditions() && reportCoverage.hasItCoveredConditions()) { + lineBuilder.setItConditions(reportCoverage.getConditions()); + lineBuilder.setItCoveredConditions(reportCoverage.getItCoveredConditions()); + } + } + + private static boolean hasIntegrationTests(BatchReport.Coverage reportCoverage){ + return reportCoverage.hasItHits() && reportCoverage.getItHits(); + } + + private static void processOverallTest(FileSourceDb.Line.Builder lineBuilder, BatchReport.Coverage reportCoverage){ + if (hasUnitTests(reportCoverage) || hasIntegrationTests(reportCoverage)) { + lineBuilder.setOverallLineHits(1); + } + if (reportCoverage.hasConditions() && reportCoverage.hasOverallCoveredConditions()) { + lineBuilder.setOverallConditions(reportCoverage.getConditions()); + lineBuilder.setOverallCoveredConditions(reportCoverage.getOverallCoveredConditions()); + } + } + + @CheckForNull + private BatchReport.Coverage getNextLineCoverageIfMatchLine(int line) { + // Get next element (if exists) + if (coverage == null && coverageIterator.hasNext()) { + coverage = coverageIterator.next(); + } + // Return current element if lines match + if (coverage != null && coverage.getLine() == line) { + return coverage; + } + return null; + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/source/LineReader.java b/server/sonar-server/src/main/java/org/sonar/server/computation/source/LineReader.java new file mode 100644 index 00000000000..ab95c60197a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/source/LineReader.java @@ -0,0 +1,29 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.source; + +import org.sonar.server.source.db.FileSourceDb; + +public interface LineReader { + + void read(FileSourceDb.Line.Builder lineBuilder); + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/source/ScmLineReader.java b/server/sonar-server/src/main/java/org/sonar/server/computation/source/ScmLineReader.java new file mode 100644 index 00000000000..4b162ff94b4 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/source/ScmLineReader.java @@ -0,0 +1,55 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.source; + +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.server.source.db.FileSourceDb; + +public class ScmLineReader implements LineReader { + + private final BatchReport.Scm scmReport; + + public ScmLineReader(BatchReport.Scm scmReport) { + this.scmReport = scmReport; + } + @Override + public void read(FileSourceDb.Line.Builder lineBuilder) { + int changeSetIndex = scmReport.getChangesetIndexByLine(lineBuilder.getLine() - 1); + BatchReport.Scm.Changeset changeset = scmReport.getChangeset(changeSetIndex); + boolean hasAuthor = changeset.hasAuthor(); + if (hasAuthor) { + lineBuilder.setScmAuthor(changeset.getAuthor()); + } + boolean hasRevision = changeset.hasRevision(); + if (hasRevision) { + lineBuilder.setScmRevision(changeset.getRevision()); + } + boolean hasDate = changeset.hasDate(); + if (hasDate) { + lineBuilder.setScmDate(changeset.getDate()); + } + + if (!hasAuthor && !hasRevision && !hasDate) { + throw new IllegalArgumentException("A changeset must contains at least one of : author, revision or date"); + } + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationSteps.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationSteps.java index 793374b6f02..9c2ce2a96db 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationSteps.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/ComputationSteps.java @@ -48,6 +48,9 @@ public class ComputationSteps { PersistEventsStep.class, PersistDuplicationMeasuresStep.class, + // TODO File sources persistence should not be activated as long as all data are not persisted and persistence should be removed from batch +// PersistFileSourcesStep.class, + // Switch snapshot and purge SwitchSnapshotStep.class, IndexComponentsStep.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistCoverageStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistCoverageStep.java deleted file mode 100644 index 28b2e9dcb27..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistCoverageStep.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package org.sonar.server.computation.step; - -import com.google.common.annotations.VisibleForTesting; -import org.sonar.api.resources.Qualifiers; -import org.sonar.batch.protocol.Constants; -import org.sonar.batch.protocol.output.BatchReport; -import org.sonar.batch.protocol.output.BatchReportReader; -import org.sonar.server.computation.ComputationContext; -import org.sonar.server.computation.source.ReportIterator; -import org.sonar.server.source.db.FileSourceDb; - -import java.io.File; - -/** - * Nothing is persist for the moment. Only Coverage are read and not persist for the moment - */ -public class PersistCoverageStep implements ComputationStep { - - // Temporary variable in order to be able to test that coverage are well computed. Will only contains data from last processed file - private FileSourceDb.Data fileSourceData; - - @Override - public String[] supportedProjectQualifiers() { - return new String[] {Qualifiers.PROJECT}; - } - - @Override - public void execute(ComputationContext context) { - int rootComponentRef = context.getReportMetadata().getRootComponentRef(); - recursivelyProcessComponent(context, rootComponentRef); - } - - private void recursivelyProcessComponent(ComputationContext context, int componentRef) { - BatchReportReader reportReader = context.getReportReader(); - BatchReport.Component component = reportReader.readComponent(componentRef); - if (component.getType().equals(Constants.ComponentType.FILE)) { - File coverageFile = reportReader.readFileCoverage(componentRef); - if (coverageFile != null) { - ReportIterator coverageReport = new ReportIterator<>(coverageFile, BatchReport.Coverage.PARSER); - processCoverage(component, coverageReport); - } - } - - for (Integer childRef : component.getChildRefList()) { - recursivelyProcessComponent(context, childRef); - } - } - - private void processCoverage(BatchReport.Component component, ReportIterator coverageReport) { - fileSourceData = null; - FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); - while (coverageReport.hasNext()) { - BatchReport.Coverage coverage = coverageReport.next(); - FileSourceDb.Line.Builder lineBuilder = dataBuilder.addLinesBuilder().setLine(coverage.getLine()); - processLineCoverage(coverage.getLine(), lineBuilder, coverage); - } - fileSourceData = dataBuilder.build(); - } - - private void processLineCoverage(int line, FileSourceDb.Line.Builder lineBuilder, BatchReport.Coverage coverage) { - // Unit test - if (coverage.getUtHits()) { - lineBuilder.setUtLineHits(1); - } - lineBuilder.setUtConditions(coverage.getConditions()); - lineBuilder.setUtCoveredConditions(coverage.getUtCoveredConditions()); - - // Integration test - if (coverage.getItHits()) { - lineBuilder.setItLineHits(1); - } - lineBuilder.setItConditions(coverage.getConditions()); - lineBuilder.setItCoveredConditions(coverage.getItCoveredConditions()); - - // Overall test - if (coverage.getUtHits() || coverage.getItHits()) { - lineBuilder.setOverallLineHits(1); - } - lineBuilder.setOverallConditions(coverage.getConditions()); - lineBuilder.setOverallCoveredConditions(coverage.getOverallCoveredConditions()); - } - - @VisibleForTesting - FileSourceDb.Data getFileSourceData() { - return fileSourceData; - } - - @Override - public String getDescription() { - return "Read Coverage"; - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java new file mode 100644 index 00000000000..6dc811fa554 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java @@ -0,0 +1,186 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.step; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.Charsets; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.LineIterator; +import org.apache.ibatis.session.ResultContext; +import org.apache.ibatis.session.ResultHandler; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; +import org.sonar.batch.protocol.Constants; +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.core.persistence.DbSession; +import org.sonar.core.persistence.MyBatis; +import org.sonar.core.source.db.FileSourceDto; +import org.sonar.server.computation.ComputationContext; +import org.sonar.server.computation.source.*; +import org.sonar.server.db.DbClient; +import org.sonar.server.source.db.FileSourceDb; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Lists.newArrayList; + +public class PersistFileSourcesStep implements ComputationStep { + + private final DbClient dbClient; + private final System2 system2; + + public PersistFileSourcesStep(DbClient dbClient, System2 system2) { + this.dbClient = dbClient; + this.system2 = system2; + } + + @Override + public String[] supportedProjectQualifiers() { + return new String[] {Qualifiers.PROJECT}; + } + + @Override + public void execute(ComputationContext context) { + int rootComponentRef = context.getReportMetadata().getRootComponentRef(); + // Don't use batch insert for file_sources since keeping all data in memory can produce OOM for big files + DbSession session = dbClient.openSession(false); + try { + final Map previousFileSourcesByUuid = new HashMap<>(); + session.select("org.sonar.core.source.db.FileSourceMapper.selectHashesForProject", context.getProject().uuid(), new ResultHandler() { + @Override + public void handleResult(ResultContext context) { + FileSourceDto dto = (FileSourceDto) context.getResultObject(); + previousFileSourcesByUuid.put(dto.getFileUuid(), dto); + } + }); + + recursivelyProcessComponent(new FileSourcesContext(session, context, previousFileSourcesByUuid), rootComponentRef); + } finally { + MyBatis.closeQuietly(session); + } + } + + private void recursivelyProcessComponent(FileSourcesContext fileSourcesContext, int componentRef) { + BatchReportReader reportReader = fileSourcesContext.context.getReportReader(); + BatchReport.Component component = reportReader.readComponent(componentRef); + if (component.getType().equals(Constants.ComponentType.FILE)) { + LineIterator linesIterator = sourceLinesIterator(reportReader.readFileSource(componentRef)); + try { + ComputeFileSourceData computeFileSourceData = new ComputeFileSourceData(linesIterator, dataLineReaders(reportReader, componentRef), component.getLines()); + ComputeFileSourceData.Data fileSourceData = computeFileSourceData.compute(); + persistSource(fileSourcesContext, fileSourceData, component); + } finally { + linesIterator.close(); + } + } + + for (Integer childRef : component.getChildRefList()) { + recursivelyProcessComponent(fileSourcesContext, childRef); + } + } + + private static LineIterator sourceLinesIterator(File file) { + try { + return IOUtils.lineIterator(FileUtils.openInputStream(file), Charsets.UTF_8); + } catch (IOException e) { + throw new IllegalStateException("Fail to traverse file: " + file, e); + } + } + + private List dataLineReaders(BatchReportReader reportReader, int componentRef) { + List lineReaders = newArrayList(); + File coverageFile = reportReader.readFileCoverage(componentRef); + if (coverageFile != null) { + lineReaders.add(new CoverageLineReader(new ReportIterator<>(coverageFile, BatchReport.Coverage.PARSER))); + } + BatchReport.Scm scmReport = reportReader.readComponentScm(componentRef); + if (scmReport != null) { + lineReaders.add(new ScmLineReader(scmReport)); + } + return lineReaders; + } + + private void persistSource(FileSourcesContext fileSourcesContext, ComputeFileSourceData.Data fileSourceData, BatchReport.Component component) { + FileSourceDb.Data fileData = fileSourceData.getFileSourceData(); + + byte[] data = FileSourceDto.encodeData(fileData); + String dataHash = DigestUtils.md5Hex(data); + String srcHash = fileSourceData.getSrcHash(); + String lineHashes = fileSourceData.getLineHashes(); + FileSourceDto previousDto = fileSourcesContext.previousFileSourcesByUuid.get(component.getUuid()); + + if (previousDto == null) { + FileSourceDto dto = new FileSourceDto() + .setProjectUuid(fileSourcesContext.context.getProject().uuid()) + .setFileUuid(component.getUuid()) + .setBinaryData(data) + .setSrcHash(srcHash) + .setDataHash(dataHash) + .setLineHashes(lineHashes) + .setCreatedAt(system2.now()) + // TODO set current date here when indexing sources in E/S will be done in this class + .setUpdatedAt(0L); + dbClient.fileSourceDao().insert(fileSourcesContext.session, dto); + fileSourcesContext.session.commit(); + } else { + // Update only if data_hash has changed or if src_hash is missing (progressive migration) + boolean binaryDataUpdated = !dataHash.equals(previousDto.getDataHash()); + boolean srcHashUpdated = !srcHash.equals(previousDto.getSrcHash()); + if (binaryDataUpdated || srcHashUpdated) { + previousDto + .setBinaryData(data) + .setDataHash(dataHash) + .setSrcHash(srcHash) + .setLineHashes(lineHashes); + // Optimization only change updated at when updating binary data to avoid unecessary indexation by E/S + if (binaryDataUpdated) { + // TODO set current date here when indexing sources in E/S will be done in this class + previousDto.setUpdatedAt(0L); + } + dbClient.fileSourceDao().update(previousDto); + fileSourcesContext.session.commit(); + } + } + } + + private static class FileSourcesContext { + DbSession session; + ComputationContext context; + Map previousFileSourcesByUuid; + + public FileSourcesContext(DbSession session, ComputationContext context, Map previousFileSourcesByUuid) { + this.context = context; + this.previousFileSourcesByUuid = previousFileSourcesByUuid; + this.session = session; + } + } + + @Override + public String getDescription() { + return "Persist file sources"; + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/source/ComputeFileSourceDataTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/source/ComputeFileSourceDataTest.java new file mode 100644 index 00000000000..920ea084310 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/source/ComputeFileSourceDataTest.java @@ -0,0 +1,110 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.source; + +import com.google.common.collect.Lists; +import org.junit.Test; +import org.sonar.server.source.db.FileSourceDb; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; + +public class ComputeFileSourceDataTest { + + @Test + public void compute_one_line() throws Exception { + ComputeFileSourceData computeFileSourceData = new ComputeFileSourceData( + newArrayList("line1").iterator(), + Lists.newArrayList(new MockLineReader()), + 1 + ); + + ComputeFileSourceData.Data data = computeFileSourceData.compute(); + assertThat(data.getSrcHash()).isEqualTo("1ddab9058a07abc0db2605ab02a61a00"); + assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\n"); + assertThat(data.getFileSourceData().getLinesList()).hasSize(1); + assertThat(data.getFileSourceData().getLines(0).getHighlighting()).isEqualTo("h-1"); + } + + @Test + public void compute_two_lines() throws Exception { + ComputeFileSourceData computeFileSourceData = new ComputeFileSourceData( + newArrayList("line1", "line2").iterator(), + Lists.newArrayList(new MockLineReader()), + 2 + ); + + ComputeFileSourceData.Data data = computeFileSourceData.compute(); + assertThat(data.getSrcHash()).isEqualTo("4fcc82a88ee38e0aa16c17f512c685c9"); + assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\ne6251bcf1a7dc3ba5e7933e325bbe605\n"); + assertThat(data.getFileSourceData().getLinesList()).hasSize(2); + assertThat(data.getFileSourceData().getLines(0).getHighlighting()).isEqualTo("h-1"); + assertThat(data.getFileSourceData().getLines(1).getHighlighting()).isEqualTo("h-2"); + } + + @Test + public void compute_missing_last_line() throws Exception { + ComputeFileSourceData computeFileSourceData = new ComputeFileSourceData( + newArrayList("line1").iterator(), + Lists.newArrayList(new MockLineReader()), + // There's only one line in the line iterator but the file has 2 lines + 2 + ); + + ComputeFileSourceData.Data data = computeFileSourceData.compute(); + assertThat(data.getSrcHash()).isEqualTo("990c2680400ef07240a32901c101768b"); + assertThat(data.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\n\n"); + assertThat(data.getFileSourceData().getLinesList()).hasSize(2); + assertThat(data.getFileSourceData().getLines(0).getHighlighting()).isEqualTo("h-1"); + assertThat(data.getFileSourceData().getLines(1).getHighlighting()).isEqualTo("h-2"); + } + + @Test + public void remove_tabs_and_spaces_in_line_hashes() throws Exception { + String refLineHashes = new ComputeFileSourceData( + newArrayList("line1").iterator(), + Lists.newArrayList(new MockLineReader()), + 1 + ).compute().getLineHashes(); + + assertThat(new ComputeFileSourceData( + newArrayList(" line\t \t 1 ").iterator(), + Lists.newArrayList(new MockLineReader()), + 1 + ).compute().getLineHashes()).isEqualTo(refLineHashes); + } + + @Test + public void compute_line_hashes_of_empty_lines() throws Exception { + assertThat(new ComputeFileSourceData( + newArrayList(" ").iterator(), + Lists.newArrayList(new MockLineReader()), + 1 + ).compute().getLineHashes()).isEqualTo("\n"); + } + + private static class MockLineReader implements LineReader { + @Override + public void read(FileSourceDb.Line.Builder lineBuilder) { + lineBuilder.setHighlighting("h-" + lineBuilder.getLine()); + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/source/CoverageLineReaderTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/source/CoverageLineReaderTest.java new file mode 100644 index 00000000000..3f89a5da323 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/source/CoverageLineReaderTest.java @@ -0,0 +1,193 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.source; + +import org.junit.Test; +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.server.source.db.FileSourceDb; + +import java.util.Collections; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; + +public class CoverageLineReaderTest { + + @Test + public void set_coverage() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList(BatchReport.Coverage.newBuilder() + .setLine(1) + .setConditions(10) + .setUtHits(true) + .setUtCoveredConditions(2) + .setItHits(true) + .setItCoveredConditions(3) + .setOverallCoveredConditions(4) + .build()).iterator()); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + computeCoverageLine.read(lineBuilder); + + assertThat(lineBuilder.getUtLineHits()).isEqualTo(1); + assertThat(lineBuilder.getUtConditions()).isEqualTo(10); + assertThat(lineBuilder.getItLineHits()).isEqualTo(1); + assertThat(lineBuilder.getItConditions()).isEqualTo(10); + assertThat(lineBuilder.getItCoveredConditions()).isEqualTo(3); + assertThat(lineBuilder.getOverallLineHits()).isEqualTo(1); + assertThat(lineBuilder.getOverallConditions()).isEqualTo(10); + } + + @Test + public void set_coverage_without_line_hits() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList(BatchReport.Coverage.newBuilder() + .setLine(1) + .setConditions(10) + .setUtHits(false) + .setUtCoveredConditions(2) + .setItHits(false) + .setItCoveredConditions(3) + .setOverallCoveredConditions(4) + .build()).iterator()); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + computeCoverageLine.read(lineBuilder); + + assertThat(lineBuilder.hasUtLineHits()).isFalse(); + assertThat(lineBuilder.hasItLineHits()).isFalse(); + assertThat(lineBuilder.hasOverallLineHits()).isFalse(); + } + + @Test + public void set_overall_line_hits_with_only_ut() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList(BatchReport.Coverage.newBuilder() + .setLine(1) + .setUtHits(true) + .setItHits(false) + .build()).iterator()); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + computeCoverageLine.read(lineBuilder); + + assertThat(lineBuilder.getOverallLineHits()).isEqualTo(1); + } + + @Test + public void set_overall_line_hits_with_only_it() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList(BatchReport.Coverage.newBuilder() + .setLine(1) + .setUtHits(false) + .setItHits(true) + .build()).iterator()); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + computeCoverageLine.read(lineBuilder); + + assertThat(lineBuilder.getOverallLineHits()).isEqualTo(1); + } + + @Test + public void set_overall_line_hits_with_ut_and_it() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList(BatchReport.Coverage.newBuilder() + .setLine(1) + .setUtHits(true) + .setItHits(true) + .build()).iterator()); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + computeCoverageLine.read(lineBuilder); + + assertThat(lineBuilder.getOverallLineHits()).isEqualTo(1); + } + + @Test + public void nothing_to_do_when_no_coverage_info() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(Collections.emptyList().iterator()); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + computeCoverageLine.read(lineBuilder); + + assertThat(lineBuilder.hasUtLineHits()).isFalse(); + assertThat(lineBuilder.hasUtConditions()).isFalse(); + assertThat(lineBuilder.hasItLineHits()).isFalse(); + assertThat(lineBuilder.hasItConditions()).isFalse(); + assertThat(lineBuilder.hasItCoveredConditions()).isFalse(); + assertThat(lineBuilder.hasOverallLineHits()).isFalse(); + assertThat(lineBuilder.hasOverallConditions()).isFalse(); + } + + @Test + public void nothing_to_do_when_no_coverage_info_for_current_line() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList( + BatchReport.Coverage.newBuilder() + .setLine(1) + .setConditions(10) + .setUtHits(true) + .setUtCoveredConditions(2) + .setItHits(true) + .setItCoveredConditions(3) + .setOverallCoveredConditions(4) + .build() + // No coverage info on line 2 + ).iterator()); + + FileSourceDb.Line.Builder line2Builder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(2); + computeCoverageLine.read(line2Builder); + + assertThat(line2Builder.hasUtLineHits()).isFalse(); + assertThat(line2Builder.hasUtConditions()).isFalse(); + assertThat(line2Builder.hasItLineHits()).isFalse(); + assertThat(line2Builder.hasItConditions()).isFalse(); + assertThat(line2Builder.hasItCoveredConditions()).isFalse(); + assertThat(line2Builder.hasOverallLineHits()).isFalse(); + assertThat(line2Builder.hasOverallConditions()).isFalse(); + } + + @Test + public void nothing_to_do_when_no_coverage_info_for_next_line() throws Exception { + CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList( + BatchReport.Coverage.newBuilder() + .setLine(1) + .setConditions(10) + .setUtHits(true) + .setUtCoveredConditions(2) + .setItHits(true) + .setItCoveredConditions(3) + .setOverallCoveredConditions(4) + .build() + // No coverage info on line 2 + ).iterator()); + + FileSourceDb.Data.Builder fileSourceBuilder = FileSourceDb.Data.newBuilder(); + FileSourceDb.Line.Builder line1Builder = fileSourceBuilder.addLinesBuilder().setLine(1); + FileSourceDb.Line.Builder line2Builder = fileSourceBuilder.addLinesBuilder().setLine(2); + computeCoverageLine.read(line1Builder); + computeCoverageLine.read(line2Builder); + + assertThat(line2Builder.hasUtLineHits()).isFalse(); + assertThat(line2Builder.hasUtConditions()).isFalse(); + assertThat(line2Builder.hasItLineHits()).isFalse(); + assertThat(line2Builder.hasItConditions()).isFalse(); + assertThat(line2Builder.hasItCoveredConditions()).isFalse(); + assertThat(line2Builder.hasOverallLineHits()).isFalse(); + assertThat(line2Builder.hasOverallConditions()).isFalse(); + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/source/ReportIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/source/ReportIteratorTest.java index 91b340b4cef..2eeab5a4693 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/source/ReportIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/source/ReportIteratorTest.java @@ -71,6 +71,16 @@ public class ReportIteratorTest { assertThat(sut.next().getLine()).isEqualTo(1); } + @Test + public void do_not_fail_when_calling_has_next_with_iterator_already_closed() throws Exception { + sut = new ReportIterator<>(file, BatchReport.Coverage.PARSER); + assertThat(sut.next().getLine()).isEqualTo(1); + assertThat(sut.hasNext()).isFalse(); + + sut.close(); + assertThat(sut.hasNext()).isFalse(); + } + @Test(expected = NoSuchElementException.class) public void test_error() throws Exception { sut = new ReportIterator<>(file, BatchReport.Coverage.PARSER); diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/source/ScmLineReaderTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/source/ScmLineReaderTest.java new file mode 100644 index 00000000000..59c27fe81e1 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/source/ScmLineReaderTest.java @@ -0,0 +1,129 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.source; + +import org.junit.Test; +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.server.source.db.FileSourceDb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class ScmLineReaderTest { + + @Test + public void set_scm() throws Exception { + BatchReport.Scm scmReport = BatchReport.Scm.newBuilder() + .addChangeset(BatchReport.Scm.Changeset.newBuilder() + .setAuthor("john") + .setDate(123456789L) + .setRevision("rev-1") + .build()) + .addChangesetIndexByLine(0) + .build(); + + ScmLineReader lineScm = new ScmLineReader(scmReport); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + lineScm.read(lineBuilder); + + assertThat(lineBuilder.getScmAuthor()).isEqualTo("john"); + assertThat(lineBuilder.getScmDate()).isEqualTo(123456789L); + assertThat(lineBuilder.getScmRevision()).isEqualTo("rev-1"); + } + + @Test + public void set_only_author() throws Exception { + BatchReport.Scm scmReport = BatchReport.Scm.newBuilder() + .addChangeset(BatchReport.Scm.Changeset.newBuilder() + .setAuthor("john") + .build()) + .addChangesetIndexByLine(0) + .build(); + + ScmLineReader lineScm = new ScmLineReader(scmReport); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + lineScm.read(lineBuilder); + + assertThat(lineBuilder.getScmAuthor()).isEqualTo("john"); + assertThat(lineBuilder.hasScmDate()).isFalse(); + assertThat(lineBuilder.hasScmRevision()).isFalse(); + } + + @Test + public void set_only_date() throws Exception { + BatchReport.Scm scmReport = BatchReport.Scm.newBuilder() + .addChangeset(BatchReport.Scm.Changeset.newBuilder() + .setDate(123456789L) + .build()) + .addChangesetIndexByLine(0) + .build(); + + ScmLineReader lineScm = new ScmLineReader(scmReport); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + lineScm.read(lineBuilder); + + assertThat(lineBuilder.hasScmAuthor()).isFalse(); + assertThat(lineBuilder.getScmDate()).isEqualTo(123456789L); + assertThat(lineBuilder.hasScmRevision()).isFalse(); + } + + @Test + public void set_only_revision() throws Exception { + BatchReport.Scm scmReport = BatchReport.Scm.newBuilder() + .addChangeset(BatchReport.Scm.Changeset.newBuilder() + .setRevision("rev-1") + .build()) + .addChangesetIndexByLine(0) + .build(); + + ScmLineReader lineScm = new ScmLineReader(scmReport); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + lineScm.read(lineBuilder); + + assertThat(lineBuilder.hasScmAuthor()).isFalse(); + assertThat(lineBuilder.hasScmDate()).isFalse(); + assertThat(lineBuilder.getScmRevision()).isEqualTo("rev-1"); + } + + @Test + public void fail_when_changeset_is_empty() throws Exception { + BatchReport.Scm scmReport = BatchReport.Scm.newBuilder() + .addChangeset(BatchReport.Scm.Changeset.newBuilder() + .build()) + .addChangesetIndexByLine(0) + .build(); + + ScmLineReader lineScm = new ScmLineReader(scmReport); + + FileSourceDb.Line.Builder lineBuilder = FileSourceDb.Data.newBuilder().addLinesBuilder().setLine(1); + try { + lineScm.read(lineBuilder); + failBecauseExceptionWasNotThrown(IllegalArgumentException.class); + } catch (IllegalArgumentException e) { + assertThat(e).hasMessage("A changeset must contains at least one of : author, revision or date"); + } + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistCoverageStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistCoverageStepTest.java deleted file mode 100644 index e42ec043ad5..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistCoverageStepTest.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package org.sonar.server.computation.step; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TemporaryFolder; -import org.sonar.batch.protocol.Constants; -import org.sonar.batch.protocol.output.BatchReport; -import org.sonar.batch.protocol.output.BatchReportReader; -import org.sonar.batch.protocol.output.BatchReportWriter; -import org.sonar.core.component.ComponentDto; -import org.sonar.server.computation.ComputationContext; -import org.sonar.server.source.db.FileSourceDb; -import org.sonar.test.DbTests; - -import java.io.File; -import java.io.IOException; - -import static com.google.common.collect.Lists.newArrayList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; - -@Category(DbTests.class) -public class PersistCoverageStepTest extends BaseStepTest { - - private static final Integer FILE_REF = 3; - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - File reportDir; - - PersistCoverageStep step; - - @Before - public void setup() throws Exception { - reportDir = temp.newFolder(); - step = new PersistCoverageStep(); - } - - @Override - protected ComputationStep step() throws IOException { - return step; - } - - @Test - public void compute_nothing() throws Exception { - initReport(); - - step.execute(new ComputationContext(new BatchReportReader(reportDir), mock(ComponentDto.class))); - - assertThat(step.getFileSourceData()).isNull(); - } - - @Test - public void compute_coverage_from_one_line() throws Exception { - BatchReportWriter writer = initReport(); - - writer.writeFileCoverage(FILE_REF, newArrayList(BatchReport.Coverage.newBuilder() - .setLine(1) - .setConditions(10) - .setUtHits(true) - .setUtCoveredConditions(2) - .setItHits(false) - .setItCoveredConditions(3) - .setOverallCoveredConditions(4) - .build())); - - step.execute(new ComputationContext(new BatchReportReader(reportDir), mock(ComponentDto.class))); - - FileSourceDb.Data data = step.getFileSourceData(); - assertThat(data.getLinesList()).hasSize(1); - - assertThat(data.getLines(0).getUtLineHits()).isEqualTo(1); - assertThat(data.getLines(0).getUtConditions()).isEqualTo(10); - assertThat(data.getLines(0).getUtCoveredConditions()).isEqualTo(2); - assertThat(data.getLines(0).hasItLineHits()).isFalse(); - assertThat(data.getLines(0).getItConditions()).isEqualTo(10); - assertThat(data.getLines(0).getItCoveredConditions()).isEqualTo(3); - assertThat(data.getLines(0).getOverallLineHits()).isEqualTo(1); - assertThat(data.getLines(0).getOverallConditions()).isEqualTo(10); - assertThat(data.getLines(0).getOverallCoveredConditions()).isEqualTo(4); - } - - @Test - public void compute_coverage_from_lines() throws Exception { - BatchReportWriter writer = initReport(); - - writer.writeFileCoverage(FILE_REF, newArrayList( - BatchReport.Coverage.newBuilder() - .setLine(1) - .setConditions(10) - .setUtHits(true) - .setUtCoveredConditions(1) - .setItHits(false) - .setItCoveredConditions(1) - .setOverallCoveredConditions(1) - .build(), - BatchReport.Coverage.newBuilder() - .setLine(2) - .setConditions(0) - .setUtHits(false) - .setUtCoveredConditions(0) - .setItHits(true) - .setItCoveredConditions(0) - .setOverallCoveredConditions(0) - .build(), - BatchReport.Coverage.newBuilder() - .setLine(3) - .setConditions(4) - .setUtHits(false) - .setUtCoveredConditions(4) - .setItHits(true) - .setItCoveredConditions(5) - .setOverallCoveredConditions(5) - .build())); - - step.execute(new ComputationContext(new BatchReportReader(reportDir), mock(ComponentDto.class))); - - FileSourceDb.Data data = step.getFileSourceData(); - assertThat(data.getLinesList()).hasSize(3); - - assertThat(data.getLines(0).getUtLineHits()).isEqualTo(1); - assertThat(data.getLines(0).hasItLineHits()).isFalse(); - - assertThat(data.getLines(1).hasUtLineHits()).isFalse(); - assertThat(data.getLines(1).getItLineHits()).isEqualTo(1); - - assertThat(data.getLines(2).hasUtLineHits()).isFalse(); - assertThat(data.getLines(2).getItLineHits()).isEqualTo(1); - } - - private BatchReportWriter initReport() { - BatchReportWriter writer = new BatchReportWriter(reportDir); - writer.writeMetadata(BatchReport.Metadata.newBuilder() - .setRootComponentRef(1) - .setProjectKey("PROJECT_KEY") - .setAnalysisDate(150000000L) - .build()); - - writer.writeComponent(BatchReport.Component.newBuilder() - .setRef(1) - .setType(Constants.ComponentType.PROJECT) - .setUuid("PROJECT_A") - .addChildRef(2) - .build()); - writer.writeComponent(BatchReport.Component.newBuilder() - .setRef(2) - .setType(Constants.ComponentType.MODULE) - .setUuid("BCDE") - .addChildRef(FILE_REF) - .build()); - writer.writeComponent(BatchReport.Component.newBuilder() - .setRef(FILE_REF) - .setType(Constants.ComponentType.FILE) - .setUuid("FILE_A") - .build()); - return writer; - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java new file mode 100644 index 00000000000..c44968ab7bb --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java @@ -0,0 +1,410 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.step; + +import com.google.common.collect.Lists; +import org.apache.commons.io.FileUtils; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.utils.System2; +import org.sonar.batch.protocol.Constants; +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.batch.protocol.output.BatchReportWriter; +import org.sonar.batch.protocol.output.FileStructure; +import org.sonar.core.persistence.DbSession; +import org.sonar.core.persistence.DbTester; +import org.sonar.core.source.db.FileSourceDao; +import org.sonar.core.source.db.FileSourceDto; +import org.sonar.server.component.ComponentTesting; +import org.sonar.server.computation.ComputationContext; +import org.sonar.server.db.DbClient; +import org.sonar.server.source.db.FileSourceDb; +import org.sonar.test.DbTests; + +import java.io.File; +import java.io.IOException; + +import static com.google.common.collect.Lists.newArrayList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Category(DbTests.class) +public class PersistFileSourcesStepTest extends BaseStepTest { + + private static final int FILE_REF = 3; + + private static final String PROJECT_UUID = "PROJECT"; + private static final String FILE_UUID = "FILE"; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + File reportDir; + + @ClassRule + public static DbTester dbTester = new DbTester(); + + DbSession session; + + DbClient dbClient; + + System2 system2; + + PersistFileSourcesStep sut; + + long now = 123456789L; + + @Before + public void setup() throws Exception { + dbTester.truncateTables(); + session = dbTester.myBatis().openSession(false); + dbClient = new DbClient(dbTester.database(), dbTester.myBatis(), new FileSourceDao(dbTester.myBatis())); + + reportDir = temp.newFolder(); + + system2 = mock(System2.class); + when(system2.now()).thenReturn(now); + sut = new PersistFileSourcesStep(dbClient, system2); + } + + @Override + protected ComputationStep step() throws IOException { + return sut; + } + + @After + public void tearDown() throws Exception { + session.close(); + } + + @Test + public void persist_sources() throws Exception { + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + .setLines(2) + .build()); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("line1", "line2")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); + assertThat(fileSourceDto.getProjectUuid()).isEqualTo(PROJECT_UUID); + assertThat(fileSourceDto.getFileUuid()).isEqualTo(FILE_UUID); + assertThat(fileSourceDto.getBinaryData()).isNotEmpty(); + assertThat(fileSourceDto.getDataHash()).isNotEmpty(); + assertThat(fileSourceDto.getLineHashes()).isNotEmpty(); + assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(0L); + + FileSourceDb.Data data = FileSourceDto.decodeData(fileSourceDto.getBinaryData()); + assertThat(data.getLinesCount()).isEqualTo(2); + assertThat(data.getLines(0).getLine()).isEqualTo(1); + assertThat(data.getLines(0).getSource()).isEqualTo("line1"); + assertThat(data.getLines(1).getLine()).isEqualTo(2); + assertThat(data.getLines(1).getSource()).isEqualTo("line2"); + } + + @Test + public void persist_last_line() throws Exception { + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + // Lines is set to 3 but only 2 lines are read from the file -> the last lines should be added + .setLines(3) + .build()); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("line1", "line2")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); + FileSourceDb.Data data = FileSourceDto.decodeData(fileSourceDto.getBinaryData()); + assertThat(data.getLinesCount()).isEqualTo(3); + assertThat(data.getLines(2).getLine()).isEqualTo(3); + assertThat(data.getLines(2).getSource()).isEmpty(); + } + + @Test + public void persist_source_hashes() throws Exception { + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + .setLines(2) + .build()); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("line\t1", "line2")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select("FILE"); + assertThat(fileSourceDto.getLineHashes()).isEqualTo("137f72c3708c6bd0de00a0e5a69c699b\ne6251bcf1a7dc3ba5e7933e325bbe605\n"); + assertThat(fileSourceDto.getSrcHash()).isEqualTo("fe1ac2747e8393015698f2724d8d2835"); + } + + @Test + public void persist_coverage() throws Exception { + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + .setLines(1) + .build()); + + writer.writeFileCoverage(FILE_REF, newArrayList(BatchReport.Coverage.newBuilder() + .setLine(1) + .setConditions(10) + .setUtHits(true) + .setUtCoveredConditions(2) + .setItHits(true) + .setItCoveredConditions(3) + .setOverallCoveredConditions(4) + .build())); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("line1")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); + FileSourceDb.Data data = FileSourceDto.decodeData(fileSourceDto.getBinaryData()); + + assertThat(data.getLinesList()).hasSize(1); + + assertThat(data.getLines(0).getUtLineHits()).isEqualTo(1); + assertThat(data.getLines(0).getUtConditions()).isEqualTo(10); + assertThat(data.getLines(0).getUtCoveredConditions()).isEqualTo(2); + assertThat(data.getLines(0).hasItLineHits()).isTrue(); + assertThat(data.getLines(0).getItConditions()).isEqualTo(10); + assertThat(data.getLines(0).getItCoveredConditions()).isEqualTo(3); + assertThat(data.getLines(0).getOverallLineHits()).isEqualTo(1); + assertThat(data.getLines(0).getOverallConditions()).isEqualTo(10); + assertThat(data.getLines(0).getOverallCoveredConditions()).isEqualTo(4); + } + + @Test + public void persist_scm() throws Exception { + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + .setLines(1) + .build()); + + writer.writeComponentScm(BatchReport.Scm.newBuilder() + .setComponentRef(FILE_REF) + .addChangeset(BatchReport.Scm.Changeset.newBuilder() + .setAuthor("john") + .setDate(123456789L) + .setRevision("rev-1") + .build()) + .addChangesetIndexByLine(0) + .build() + ); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("line1")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); + FileSourceDb.Data data = FileSourceDto.decodeData(fileSourceDto.getBinaryData()); + + assertThat(data.getLinesList()).hasSize(1); + + assertThat(data.getLines(0).getScmAuthor()).isEqualTo("john"); + assertThat(data.getLines(0).getScmDate()).isEqualTo(123456789L); + assertThat(data.getLines(0).getScmRevision()).isEqualTo("rev-1"); + } + + @Test + public void not_update_sources_when_nothing_has_changed() throws Exception { + // Existing sources + long past = 150000L; + String srcHash = "5b4bd9815cdb17b8ceae19eb1810c34c"; + String lineHashes = "6438c669e0d0de98e6929c2cc0fac474\n"; + String dataHash = "6cad150e3d065976c230cddc5a09efaa"; + + dbClient.fileSourceDao().insert(session, new FileSourceDto() + .setProjectUuid(PROJECT_UUID) + .setFileUuid(FILE_UUID) + .setSrcHash(srcHash) + .setLineHashes(lineHashes) + .setDataHash(dataHash) + .setBinaryData(FileSourceDto.encodeData(FileSourceDb.Data.newBuilder() + .addLines(FileSourceDb.Line.newBuilder() + .setLine(1) + .setSource("line") + .build()) + .build())) + .setCreatedAt(past) + .setUpdatedAt(past)); + session.commit(); + + // Sources from the report + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + .setLines(1) + .build()); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("line")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); + assertThat(fileSourceDto.getSrcHash()).isEqualTo(srcHash); + assertThat(fileSourceDto.getLineHashes()).isEqualTo(lineHashes); + assertThat(fileSourceDto.getDataHash()).isEqualTo(dataHash); + assertThat(fileSourceDto.getCreatedAt()).isEqualTo(past); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(past); + } + + @Test + public void update_sources_when_source_updated() throws Exception { + // Existing sources + long past = 150000L; + dbClient.fileSourceDao().insert(session, new FileSourceDto() + .setProjectUuid(PROJECT_UUID) + .setFileUuid(FILE_UUID) + .setSrcHash("5b4bd9815cdb17b8ceae19eb1810c34c") + .setLineHashes("6438c669e0d0de98e6929c2cc0fac474\n") + .setDataHash("6cad150e3d065976c230cddc5a09efaa") + .setBinaryData(FileSourceDto.encodeData(FileSourceDb.Data.newBuilder() + .addLines(FileSourceDb.Line.newBuilder() + .setLine(1) + .setSource("line") + .build()) + .build())) + .setCreatedAt(past) + .setUpdatedAt(past)); + session.commit(); + + // New sources from the report + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + .setLines(1) + .build()); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("new line")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); + assertThat(fileSourceDto.getCreatedAt()).isEqualTo(past); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(0L); + } + + @Test + public void update_sources_when_src_hash_is_missing() throws Exception { + // Existing sources + long past = 150000L; + dbClient.fileSourceDao().insert(session, new FileSourceDto() + .setProjectUuid(PROJECT_UUID) + .setFileUuid(FILE_UUID) + // Source hash is missing, udate will be made + .setLineHashes("6438c669e0d0de98e6929c2cc0fac474\n") + .setDataHash("6cad150e3d065976c230cddc5a09efaa") + .setBinaryData(FileSourceDto.encodeData(FileSourceDb.Data.newBuilder() + .addLines(FileSourceDb.Line.newBuilder() + .setLine(1) + .setSource("line") + .build()) + .build())) + .setCreatedAt(past) + .setUpdatedAt(past)); + session.commit(); + + // New sources from the report + BatchReportWriter writer = initReport(); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(FILE_REF) + .setType(Constants.ComponentType.FILE) + .setUuid(FILE_UUID) + .setLines(1) + .build()); + + File sourceFile = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, FILE_REF); + FileUtils.writeLines(sourceFile, Lists.newArrayList("line")); + + sut.execute(new ComputationContext(new BatchReportReader(reportDir), ComponentTesting.newProjectDto(PROJECT_UUID))); + + assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); + FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); + assertThat(fileSourceDto.getCreatedAt()).isEqualTo(past); + // Updated at is not updated to not reindex the file source in E/S as the src hash is not indexed + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(past); + assertThat(fileSourceDto.getSrcHash()).isEqualTo("5b4bd9815cdb17b8ceae19eb1810c34c"); + } + + private BatchReportWriter initReport() throws IOException { + BatchReportWriter writer = new BatchReportWriter(reportDir); + writer.writeMetadata(BatchReport.Metadata.newBuilder() + .setRootComponentRef(1) + .setProjectKey("PROJECT_KEY") + .build()); + + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(1) + .setType(Constants.ComponentType.PROJECT) + .setUuid(PROJECT_UUID) + .addChildRef(2) + .build()); + writer.writeComponent(BatchReport.Component.newBuilder() + .setRef(2) + .setType(Constants.ComponentType.MODULE) + .setUuid("MODULE") + .addChildRef(FILE_REF) + .build()); + + return writer; + } + +} diff --git a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java index 16864674f20..cbd920455e9 100644 --- a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java +++ b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java @@ -104,23 +104,31 @@ public class FileSourceDao implements BatchComponent, ServerComponent, DaoCompon public void insert(FileSourceDto dto) { DbSession session = mybatis.openSession(false); try { - session.getMapper(FileSourceMapper.class).insert(dto); + insert(session, dto); session.commit(); } finally { MyBatis.closeQuietly(session); } } + public void insert(DbSession session, FileSourceDto dto) { + session.getMapper(FileSourceMapper.class).insert(dto); + } + public void update(FileSourceDto dto) { DbSession session = mybatis.openSession(false); try { - session.getMapper(FileSourceMapper.class).update(dto); + update(session, dto); session.commit(); } finally { MyBatis.closeQuietly(session); } } + public void update(DbSession session, FileSourceDto dto) { + session.getMapper(FileSourceMapper.class).update(dto); + } + public void updateDateWhenUpdatedDateIsZero(DbSession session, String projectUuid, long updateDate) { session.getMapper(FileSourceMapper.class).updateDateWhenUpdatedDateIsZero(projectUuid, updateDate); } -- 2.39.5