source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2022 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.ce.task.projectanalysis.step;
  21
  22 import java.util.ArrayList;
  23 import java.util.List;
  24 import java.util.Map;
  25 import org.sonar.api.utils.System2;
  26 import org.sonar.ce.task.projectanalysis.issue.ProtoIssueCache;
  27 import org.sonar.ce.task.projectanalysis.issue.RuleRepository;
  28 import org.sonar.ce.task.projectanalysis.issue.UpdateConflictResolver;
  29 import org.sonar.ce.task.step.ComputationStep;
  30 import org.sonar.core.issue.DefaultIssue;
  31 import org.sonar.core.util.CloseableIterator;
  32 import org.sonar.core.util.UuidFactory;
  33 import org.sonar.db.BatchSession;
  34 import org.sonar.db.DbClient;
  35 import org.sonar.db.DbSession;
  36 import org.sonar.db.issue.IssueChangeMapper;
  37 import org.sonar.db.issue.IssueDto;
  38 import org.sonar.db.issue.IssueMapper;
  39 import org.sonar.db.issue.NewCodeReferenceIssueDto;
  40 import org.sonar.server.issue.IssueStorage;
  41
  42 import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
  43 import static org.sonar.core.util.stream.MoreCollectors.toList;
  44 import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex;
  45
  46 public class PersistIssuesStep implements ComputationStep {
  47   // holding up to 1000 DefaultIssue (max size of addedIssues and updatedIssues at any given time) in memory should not
  48   // be a problem while making sure we leverage extensively the batch feature to speed up persistence
  49   private static final int ISSUE_BATCHING_SIZE = BatchSession.MAX_BATCH_SIZE * 2;
  50
  51   private final DbClient dbClient;
  52   private final System2 system2;
  53   private final UpdateConflictResolver conflictResolver;
  54   private final RuleRepository ruleRepository;
  55   private final ProtoIssueCache protoIssueCache;
  56   private final IssueStorage issueStorage;
  57   private final UuidFactory uuidFactory;
  58
  59   public PersistIssuesStep(DbClient dbClient, System2 system2, UpdateConflictResolver conflictResolver,
  60     RuleRepository ruleRepository, ProtoIssueCache protoIssueCache, IssueStorage issueStorage, UuidFactory uuidFactory) {
  61     this.dbClient = dbClient;
  62     this.system2 = system2;
  63     this.conflictResolver = conflictResolver;
  64     this.ruleRepository = ruleRepository;
  65     this.protoIssueCache = protoIssueCache;
  66     this.issueStorage = issueStorage;
  67     this.uuidFactory = uuidFactory;
  68   }
  69
  70   @Override
  71   public void execute(ComputationStep.Context context) {
  72     context.getStatistics().add("cacheSize", humanReadableByteCountSI(protoIssueCache.fileSize()));
  73     IssueStatistics statistics = new IssueStatistics();
  74     try (DbSession dbSession = dbClient.openSession(true);
  75          CloseableIterator<DefaultIssue> issues = protoIssueCache.traverse()) {
  76       List<DefaultIssue> addedIssues = new ArrayList<>(ISSUE_BATCHING_SIZE);
  77       List<DefaultIssue> updatedIssues = new ArrayList<>(ISSUE_BATCHING_SIZE);
  78       List<DefaultIssue> noLongerNewIssues = new ArrayList<>(ISSUE_BATCHING_SIZE);
  79
  80       IssueMapper mapper = dbSession.getMapper(IssueMapper.class);
  81       IssueChangeMapper changeMapper = dbSession.getMapper(IssueChangeMapper.class);
  82       while (issues.hasNext()) {
  83         DefaultIssue issue = issues.next();
  84         if (issue.isNew() || issue.isCopied()) {
  85           addedIssues.add(issue);
  86           if (addedIssues.size() >= ISSUE_BATCHING_SIZE) {
  87             persistNewIssues(statistics, addedIssues, mapper, changeMapper);
  88             addedIssues.clear();
  89           }
  90         } else if (issue.isChanged()) {
  91           updatedIssues.add(issue);
  92           if (updatedIssues.size() >= ISSUE_BATCHING_SIZE) {
  93             persistUpdatedIssues(statistics, updatedIssues, mapper, changeMapper);
  94             updatedIssues.clear();
  95           }
  96         } else if (issue.isNoLongerNewCodeReferenceIssue()) {
  97           noLongerNewIssues.add(issue);
  98           if (noLongerNewIssues.size() >= ISSUE_BATCHING_SIZE) {
  99             persistNoLongerNewIssues(statistics, noLongerNewIssues, mapper);
 100             noLongerNewIssues.clear();
 101           }
 102         }
 103       }
 104       persistNewIssues(statistics, addedIssues, mapper, changeMapper);
 105       persistUpdatedIssues(statistics, updatedIssues, mapper, changeMapper);
 106       persistNoLongerNewIssues(statistics, noLongerNewIssues, mapper);
 107       flushSession(dbSession);
 108     } finally {
 109       statistics.dumpTo(context);
 110     }
 111   }
 112
 113   private void persistNewIssues(IssueStatistics statistics, List<DefaultIssue> addedIssues, IssueMapper mapper, IssueChangeMapper changeMapper) {
 114     if (addedIssues.isEmpty()) {
 115       return;
 116     }
 117
 118     long now = system2.now();
 119     addedIssues.forEach(i -> {
 120       String ruleUuid = ruleRepository.getByKey(i.ruleKey()).getUuid();
 121       IssueDto dto = IssueDto.toDtoForComputationInsert(i, ruleUuid, now);
 122       mapper.insert(dto);
 123       if (i.isOnReferencedBranch() && i.isOnChangedLine()) {
 124         mapper.insertAsNewCodeOnReferenceBranch(NewCodeReferenceIssueDto.fromIssueDto(dto, now, uuidFactory));
 125       }
 126       statistics.inserts++;
 127     });
 128
 129     addedIssues.forEach(i -> issueStorage.insertChanges(changeMapper, i, uuidFactory));
 130   }
 131
 132   private void persistUpdatedIssues(IssueStatistics statistics, List<DefaultIssue> updatedIssues, IssueMapper mapper, IssueChangeMapper changeMapper) {
 133     if (updatedIssues.isEmpty()) {
 134       return;
 135     }
 136
 137     long now = system2.now();
 138     updatedIssues.forEach(i -> {
 139       IssueDto dto = IssueDto.toDtoForUpdate(i, now);
 140       mapper.updateIfBeforeSelectedDate(dto);
 141       statistics.updates++;
 142     });
 143
 144     // retrieve those of the updatedIssues which have not been updated and apply conflictResolver on them
 145     List<String> updatedIssueKeys = updatedIssues.stream().map(DefaultIssue::key).collect(toList(updatedIssues.size()));
 146     List<IssueDto> conflictIssueKeys = mapper.selectByKeysIfNotUpdatedAt(updatedIssueKeys, now);
 147     if (!conflictIssueKeys.isEmpty()) {
 148       Map<String, DefaultIssue> issuesByKeys = updatedIssues.stream().collect(uniqueIndex(DefaultIssue::key, updatedIssues.size()));
 149       conflictIssueKeys
 150         .forEach(dbIssue -> {
 151           DefaultIssue updatedIssue = issuesByKeys.get(dbIssue.getKey());
 152           conflictResolver.resolve(updatedIssue, dbIssue, mapper);
 153           statistics.merged++;
 154         });
 155     }
 156
 157     updatedIssues.forEach(i -> issueStorage.insertChanges(changeMapper, i, uuidFactory));
 158   }
 159
 160   private static void persistNoLongerNewIssues(IssueStatistics statistics, List<DefaultIssue> noLongerNewIssues, IssueMapper mapper) {
 161     if (noLongerNewIssues.isEmpty()) {
 162       return;
 163     }
 164
 165     noLongerNewIssues.forEach(i -> {
 166       mapper.deleteAsNewCodeOnReferenceBranch(i.key());
 167       statistics.updates++;
 168     });
 169
 170   }
 171
 172   private static void flushSession(DbSession dbSession) {
 173     dbSession.flushStatements();
 174     dbSession.commit();
 175   }
 176
 177   @Override
 178   public String getDescription() {
 179     return "Persist issues";
 180   }
 181
 182   private static class IssueStatistics {
 183     private int inserts = 0;
 184     private int updates = 0;
 185     private int merged = 0;
 186
 187     private void dumpTo(ComputationStep.Context context) {
 188       context.getStatistics()
 189         .add("inserts", String.valueOf(inserts))
 190         .add("updates", String.valueOf(updates))
 191         .add("merged", String.valueOf(merged));
 192     }
 193   }
 194 }