source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2016 SonarSource SA
   4  * mailto:contact AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.server.computation.task.projectanalysis.step;
  21
  22 import com.google.common.base.Optional;
  23 import com.google.common.collect.ImmutableMap;
  24 import java.util.ArrayList;
  25 import java.util.HashMap;
  26 import java.util.List;
  27 import java.util.Map;
  28 import javax.annotation.CheckForNull;
  29 import javax.annotation.Nullable;
  30 import org.apache.commons.codec.digest.DigestUtils;
  31 import org.apache.commons.lang.ObjectUtils;
  32 import org.apache.ibatis.session.ResultContext;
  33 import org.apache.ibatis.session.ResultHandler;
  34 import org.sonar.api.utils.System2;
  35 import org.sonar.core.util.CloseableIterator;
  36 import org.sonar.db.DbClient;
  37 import org.sonar.db.DbSession;
  38 import org.sonar.db.MyBatis;
  39 import org.sonar.db.protobuf.DbFileSources;
  40 import org.sonar.db.source.FileSourceDto;
  41 import org.sonar.db.source.FileSourceDto.Type;
  42 import org.sonar.scanner.protocol.output.ScannerReport;
  43 import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReader;
  44 import org.sonar.server.computation.task.projectanalysis.component.Component;
  45 import org.sonar.server.computation.task.projectanalysis.component.CrawlerDepthLimit;
  46 import org.sonar.server.computation.task.projectanalysis.component.DepthTraversalTypeAwareCrawler;
  47 import org.sonar.server.computation.task.projectanalysis.component.TreeRootHolder;
  48 import org.sonar.server.computation.task.projectanalysis.component.TypeAwareVisitorAdapter;
  49 import org.sonar.server.computation.task.projectanalysis.duplication.DuplicationRepository;
  50 import org.sonar.server.computation.task.projectanalysis.scm.Changeset;
  51 import org.sonar.server.computation.task.projectanalysis.scm.ScmInfo;
  52 import org.sonar.server.computation.task.projectanalysis.scm.ScmInfoRepository;
  53 import org.sonar.server.computation.task.projectanalysis.source.ComputeFileSourceData;
  54 import org.sonar.server.computation.task.projectanalysis.source.CoverageLineReader;
  55 import org.sonar.server.computation.task.projectanalysis.source.DuplicationLineReader;
  56 import org.sonar.server.computation.task.projectanalysis.source.HighlightingLineReader;
  57 import org.sonar.server.computation.task.projectanalysis.source.LineReader;
  58 import org.sonar.server.computation.task.projectanalysis.source.RangeOffsetConverter;
  59 import org.sonar.server.computation.task.projectanalysis.source.ScmLineReader;
  60 import org.sonar.server.computation.task.projectanalysis.source.SourceLinesRepository;
  61 import org.sonar.server.computation.task.projectanalysis.source.SymbolsLineReader;
  62 import org.sonar.server.computation.task.step.ComputationStep;
  63
  64 import static org.sonar.server.computation.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER;
  65
  66 public class PersistFileSourcesStep implements ComputationStep {
  67
  68   private final DbClient dbClient;
  69   private final System2 system2;
  70   private final TreeRootHolder treeRootHolder;
  71   private final BatchReportReader reportReader;
  72   private final SourceLinesRepository sourceLinesRepository;
  73   private final ScmInfoRepository scmInfoRepository;
  74   private final DuplicationRepository duplicationRepository;
  75
  76   public PersistFileSourcesStep(DbClient dbClient, System2 system2, TreeRootHolder treeRootHolder, BatchReportReader reportReader, SourceLinesRepository sourceLinesRepository,
  77     ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository) {
  78     this.dbClient = dbClient;
  79     this.system2 = system2;
  80     this.treeRootHolder = treeRootHolder;
  81     this.reportReader = reportReader;
  82     this.sourceLinesRepository = sourceLinesRepository;
  83     this.scmInfoRepository = scmInfoRepository;
  84     this.duplicationRepository = duplicationRepository;
  85   }
  86
  87   @Override
  88   public void execute() {
  89     // Don't use batch insert for file_sources since keeping all data in memory can produce OOM for big files
  90     DbSession session = dbClient.openSession(false);
  91     try {
  92       new DepthTraversalTypeAwareCrawler(new FileSourceVisitor(session))
  93         .visit(treeRootHolder.getRoot());
  94     } finally {
  95       MyBatis.closeQuietly(session);
  96     }
  97   }
  98
  99   private class FileSourceVisitor extends TypeAwareVisitorAdapter {
 100
 101     private final DbSession session;
 102
 103     private Map<String, FileSourceDto> previousFileSourcesByUuid = new HashMap<>();
 104     private String projectUuid;
 105
 106     private FileSourceVisitor(DbSession session) {
 107       super(CrawlerDepthLimit.FILE, PRE_ORDER);
 108       this.session = session;
 109     }
 110
 111     @Override
 112     public void visitProject(Component project) {
 113       this.projectUuid = project.getUuid();
 114       session.select("org.sonar.db.source.FileSourceMapper.selectHashesForProject", ImmutableMap.of("projectUuid", projectUuid, "dataType", Type.SOURCE),
 115         new ResultHandler() {
 116           @Override
 117           public void handleResult(ResultContext context) {
 118             FileSourceDto dto = (FileSourceDto) context.getResultObject();
 119             previousFileSourcesByUuid.put(dto.getFileUuid(), dto);
 120           }
 121         });
 122     }
 123
 124     @Override
 125     public void visitFile(Component file) {
 126       int fileRef = file.getReportAttributes().getRef();
 127       ScannerReport.Component component = reportReader.readComponent(fileRef);
 128       CloseableIterator<String> linesIterator = sourceLinesRepository.readLines(file);
 129       LineReaders lineReaders = new LineReaders(reportReader, scmInfoRepository, duplicationRepository, file);
 130       try {
 131         ComputeFileSourceData computeFileSourceData = new ComputeFileSourceData(linesIterator, lineReaders.readers(), component.getLines());
 132         ComputeFileSourceData.Data fileSourceData = computeFileSourceData.compute();
 133         persistSource(fileSourceData, file.getUuid(), lineReaders.getLatestChange());
 134       } catch (Exception e) {
 135         throw new IllegalStateException(String.format("Cannot persist sources of %s", file.getKey()), e);
 136       } finally {
 137         linesIterator.close();
 138         lineReaders.close();
 139       }
 140     }
 141
 142     private void persistSource(ComputeFileSourceData.Data fileSourceData, String componentUuid, @Nullable Changeset latestChange) {
 143       DbFileSources.Data fileData = fileSourceData.getFileSourceData();
 144
 145       byte[] data = FileSourceDto.encodeSourceData(fileData);
 146       String dataHash = DigestUtils.md5Hex(data);
 147       String srcHash = fileSourceData.getSrcHash();
 148       String lineHashes = fileSourceData.getLineHashes();
 149       FileSourceDto previousDto = previousFileSourcesByUuid.get(componentUuid);
 150
 151       if (previousDto == null) {
 152         FileSourceDto dto = new FileSourceDto()
 153           .setProjectUuid(projectUuid)
 154           .setFileUuid(componentUuid)
 155           .setDataType(Type.SOURCE)
 156           .setBinaryData(data)
 157           .setSrcHash(srcHash)
 158           .setDataHash(dataHash)
 159           .setLineHashes(lineHashes)
 160           .setCreatedAt(system2.now())
 161           .setUpdatedAt(system2.now())
 162           .setRevision(computeRevision(latestChange));
 163         dbClient.fileSourceDao().insert(session, dto);
 164         session.commit();
 165       } else {
 166         // Update only if data_hash has changed or if src_hash is missing or revision is missing (progressive migration)
 167         boolean binaryDataUpdated = !dataHash.equals(previousDto.getDataHash());
 168         boolean srcHashUpdated = !srcHash.equals(previousDto.getSrcHash());
 169         String revision = computeRevision(latestChange);
 170         boolean revisionUpdated = !ObjectUtils.equals(revision, previousDto.getRevision());
 171         if (binaryDataUpdated || srcHashUpdated || revisionUpdated) {
 172           previousDto
 173             .setBinaryData(data)
 174             .setDataHash(dataHash)
 175             .setSrcHash(srcHash)
 176             .setLineHashes(lineHashes)
 177             .setRevision(revision)
 178             .setUpdatedAt(system2.now());
 179           dbClient.fileSourceDao().update(session, previousDto);
 180           session.commit();
 181         }
 182       }
 183     }
 184
 185     @CheckForNull
 186     private String computeRevision(@Nullable Changeset latestChange) {
 187       if (latestChange == null) {
 188         return null;
 189       }
 190       return latestChange.getRevision();
 191     }
 192   }
 193
 194   private static class LineReaders {
 195     private final List<LineReader> readers = new ArrayList<>();
 196     private final List<CloseableIterator<?>> closeables = new ArrayList<>();
 197     @CheckForNull
 198     private final ScmLineReader scmLineReader;
 199
 200     LineReaders(BatchReportReader reportReader, ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository, Component component) {
 201       int componentRef = component.getReportAttributes().getRef();
 202       CloseableIterator<ScannerReport.LineCoverage> coverageIt = reportReader.readComponentCoverage(componentRef);
 203       closeables.add(coverageIt);
 204       readers.add(new CoverageLineReader(coverageIt));
 205
 206       Optional<ScmInfo> scmInfoOptional = scmInfoRepository.getScmInfo(component);
 207       if (scmInfoOptional.isPresent()) {
 208         this.scmLineReader = new ScmLineReader(scmInfoOptional.get());
 209         readers.add(scmLineReader);
 210       } else {
 211         this.scmLineReader = null;
 212       }
 213
 214       RangeOffsetConverter rangeOffsetConverter = new RangeOffsetConverter();
 215       CloseableIterator<ScannerReport.SyntaxHighlightingRule> highlightingIt = reportReader.readComponentSyntaxHighlighting(componentRef);
 216       closeables.add(highlightingIt);
 217       readers.add(new HighlightingLineReader(component, highlightingIt, rangeOffsetConverter));
 218
 219       CloseableIterator<ScannerReport.Symbol> symbolsIt = reportReader.readComponentSymbols(componentRef);
 220       closeables.add(symbolsIt);
 221       readers.add(new SymbolsLineReader(component, symbolsIt, rangeOffsetConverter));
 222
 223       readers.add(new DuplicationLineReader(duplicationRepository.getDuplications(component)));
 224     }
 225
 226     List<LineReader> readers() {
 227       return readers;
 228     }
 229
 230     void close() {
 231       for (CloseableIterator<?> reportIterator : closeables) {
 232         reportIterator.close();
 233       }
 234     }
 235
 236     @CheckForNull
 237     public Changeset getLatestChange() {
 238       if (scmLineReader == null) {
 239         return null;
 240       }
 241       return scmLineReader.getLatestChange();
 242     }
 243   }
 244
 245   @Override
 246   public String getDescription() {
 247     return "Persist sources";
 248   }
 249 }