source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2024 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.ce.task.projectanalysis.step;
  21
  22 import java.util.ArrayList;
  23 import java.util.LinkedList;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.function.Function;
  27 import java.util.stream.Collectors;
  28 import org.sonar.api.utils.System2;
  29 import org.sonar.ce.task.projectanalysis.issue.ProtoIssueCache;
  30 import org.sonar.ce.task.projectanalysis.issue.RuleRepository;
  31 import org.sonar.ce.task.projectanalysis.issue.UpdateConflictResolver;
  32 import org.sonar.ce.task.projectanalysis.period.PeriodHolder;
  33 import org.sonar.ce.task.step.ComputationStep;
  34 import org.sonar.core.issue.DefaultIssue;
  35 import org.sonar.core.util.CloseableIterator;
  36 import org.sonar.core.util.UuidFactory;
  37 import org.sonar.db.BatchSession;
  38 import org.sonar.db.DbClient;
  39 import org.sonar.db.DbSession;
  40 import org.sonar.db.issue.AnticipatedTransitionMapper;
  41 import org.sonar.db.issue.IssueChangeMapper;
  42 import org.sonar.db.issue.IssueDao;
  43 import org.sonar.db.issue.IssueDto;
  44 import org.sonar.db.issue.NewCodeReferenceIssueDto;
  45 import org.sonar.db.newcodeperiod.NewCodePeriodType;
  46 import org.sonar.server.issue.IssueStorage;
  47
  48 import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
  49
  50 public class PersistIssuesStep implements ComputationStep {
  51   // holding up to 1000 DefaultIssue (max size of addedIssues and updatedIssues at any given time) in memory should not
  52   // be a problem while making sure we leverage extensively the batch feature to speed up persistence
  53   private static final int ISSUE_BATCHING_SIZE = BatchSession.MAX_BATCH_SIZE * 2;
  54
  55   private final DbClient dbClient;
  56   private final System2 system2;
  57   private final UpdateConflictResolver conflictResolver;
  58   private final RuleRepository ruleRepository;
  59   private final PeriodHolder periodHolder;
  60   private final ProtoIssueCache protoIssueCache;
  61   private final IssueStorage issueStorage;
  62   private final UuidFactory uuidFactory;
  63
  64   public PersistIssuesStep(DbClient dbClient, System2 system2, UpdateConflictResolver conflictResolver,
  65     RuleRepository ruleRepository, PeriodHolder periodHolder, ProtoIssueCache protoIssueCache, IssueStorage issueStorage,
  66     UuidFactory uuidFactory) {
  67     this.dbClient = dbClient;
  68     this.system2 = system2;
  69     this.conflictResolver = conflictResolver;
  70     this.ruleRepository = ruleRepository;
  71     this.periodHolder = periodHolder;
  72     this.protoIssueCache = protoIssueCache;
  73     this.issueStorage = issueStorage;
  74     this.uuidFactory = uuidFactory;
  75   }
  76
  77   @Override
  78   public void execute(ComputationStep.Context context) {
  79     context.getStatistics().add("cacheSize", humanReadableByteCountSI(protoIssueCache.fileSize()));
  80     IssueStatistics statistics = new IssueStatistics();
  81     try (DbSession dbSession = dbClient.openSession(true);
  82       CloseableIterator<DefaultIssue> issues = protoIssueCache.traverse()) {
  83       List<DefaultIssue> addedIssues = new ArrayList<>(ISSUE_BATCHING_SIZE);
  84       List<DefaultIssue> updatedIssues = new ArrayList<>(ISSUE_BATCHING_SIZE);
  85       List<DefaultIssue> noLongerNewIssues = new ArrayList<>(ISSUE_BATCHING_SIZE);
  86       List<DefaultIssue> newCodeIssuesToMigrate = new ArrayList<>(ISSUE_BATCHING_SIZE);
  87       IssueDao issueDao = dbClient.issueDao();
  88       IssueChangeMapper changeMapper = dbSession.getMapper(IssueChangeMapper.class);
  89       AnticipatedTransitionMapper anticipatedTransitionMapper = dbSession.getMapper(AnticipatedTransitionMapper.class);
  90       while (issues.hasNext()) {
  91         DefaultIssue issue = issues.next();
  92         if (issue.isNew() || issue.isCopied()) {
  93           addedIssues.add(issue);
  94           if (addedIssues.size() >= ISSUE_BATCHING_SIZE) {
  95             persistNewIssues(statistics, addedIssues, issueDao, changeMapper, anticipatedTransitionMapper, dbSession);
  96             addedIssues.clear();
  97           }
  98         } else if (issue.isChanged()) {
  99           updatedIssues.add(issue);
 100           if (updatedIssues.size() >= ISSUE_BATCHING_SIZE) {
 101             persistUpdatedIssues(statistics, updatedIssues, issueDao, changeMapper, dbSession);
 102             updatedIssues.clear();
 103           }
 104         } else if (isOnBranchUsingReferenceBranch() && issue.isNoLongerNewCodeReferenceIssue()) {
 105           noLongerNewIssues.add(issue);
 106           if (noLongerNewIssues.size() >= ISSUE_BATCHING_SIZE) {
 107             persistNoLongerNewIssues(statistics, noLongerNewIssues, issueDao, dbSession);
 108             noLongerNewIssues.clear();
 109           }
 110         } else if (isOnBranchUsingReferenceBranch() && issue.isToBeMigratedAsNewCodeReferenceIssue()) {
 111           newCodeIssuesToMigrate.add(issue);
 112           if (newCodeIssuesToMigrate.size() >= ISSUE_BATCHING_SIZE) {
 113             persistNewCodeIssuesToMigrate(statistics, newCodeIssuesToMigrate, issueDao, dbSession);
 114             newCodeIssuesToMigrate.clear();
 115           }
 116         }
 117       }
 118
 119       persistNewIssues(statistics, addedIssues, issueDao, changeMapper, anticipatedTransitionMapper, dbSession);
 120       persistUpdatedIssues(statistics, updatedIssues, issueDao, changeMapper, dbSession);
 121       persistNoLongerNewIssues(statistics, noLongerNewIssues, issueDao, dbSession);
 122       persistNewCodeIssuesToMigrate(statistics, newCodeIssuesToMigrate, issueDao, dbSession);
 123       flushSession(dbSession);
 124     } finally {
 125       statistics.dumpTo(context);
 126     }
 127   }
 128
 129   private void persistNewIssues(IssueStatistics statistics, List<DefaultIssue> addedIssues,
 130     IssueDao issueDao, IssueChangeMapper changeMapper, AnticipatedTransitionMapper anticipatedTransitionMapper, DbSession dbSession) {
 131
 132     final long now = system2.now();
 133
 134     List<IssueDto> issueDtos = new LinkedList<>();
 135     addedIssues.forEach(addedIssue -> {
 136       String ruleUuid = ruleRepository.getByKey(addedIssue.ruleKey()).getUuid();
 137       IssueDto dto = IssueDto.toDtoForComputationInsert(addedIssue, ruleUuid, now);
 138       issueDao.insertWithoutImpacts(dbSession, dto);
 139       issueDtos.add(dto);
 140       if (isOnBranchUsingReferenceBranch() && addedIssue.isOnChangedLine()) {
 141         issueDao.insertAsNewCodeOnReferenceBranch(dbSession, NewCodeReferenceIssueDto.fromIssueDto(dto, now, uuidFactory));
 142       }
 143       statistics.inserts++;
 144       issueStorage.insertChanges(changeMapper, addedIssue, uuidFactory);
 145       addedIssue.getAnticipatedTransitionUuid().ifPresent(anticipatedTransitionMapper::delete);
 146     });
 147
 148     issueDtos.forEach(issueDto -> issueDao.insertIssueImpacts(dbSession, issueDto));
 149   }
 150
 151   private void persistUpdatedIssues(IssueStatistics statistics, List<DefaultIssue> updatedIssues, IssueDao issueDao,
 152     IssueChangeMapper changeMapper, DbSession dbSession) {
 153     if (updatedIssues.isEmpty()) {
 154       return;
 155     }
 156
 157     long now = system2.now();
 158     LinkedList<IssueDto> updatedIssueDtos = new LinkedList<>();
 159     updatedIssues.forEach(i -> {
 160       IssueDto dto = IssueDto.toDtoForUpdate(i, now);
 161       boolean isUpdated = issueDao.updateIfBeforeSelectedDate(dbSession, dto);
 162       if (isUpdated) {
 163         updatedIssueDtos.add(dto);
 164       }
 165       statistics.updates++;
 166     });
 167     updatedIssueDtos.forEach(i -> issueDao.deleteIssueImpacts(dbSession, i));
 168     updatedIssueDtos.forEach(i -> issueDao.insertIssueImpacts(dbSession, i));
 169
 170     // retrieve those of the updatedIssues which have not been updated and apply conflictResolver on them
 171     List<String> updatedIssueKeys = updatedIssues.stream().map(DefaultIssue::key).toList();
 172     List<IssueDto> conflictIssueKeys = issueDao.selectByKeysIfNotUpdatedAt(dbSession, updatedIssueKeys, now);
 173     if (!conflictIssueKeys.isEmpty()) {
 174       updatedIssueDtos.clear();
 175       Map<String, DefaultIssue> issuesByKeys = updatedIssues.stream().collect(Collectors.toMap(DefaultIssue::key, Function.identity()));
 176       conflictIssueKeys
 177         .forEach(dbIssue -> {
 178           DefaultIssue updatedIssue = issuesByKeys.get(dbIssue.getKey());
 179           IssueDto issueToBeUpdated = conflictResolver.resolve(updatedIssue, dbIssue);
 180           issueDao.updateWithoutIssueImpacts(dbSession, issueToBeUpdated);
 181           updatedIssueDtos.add(issueToBeUpdated);
 182           statistics.merged++;
 183         });
 184
 185       updatedIssueDtos.forEach(i -> issueDao.deleteIssueImpacts(dbSession, i));
 186       updatedIssueDtos.forEach(i -> issueDao.insertIssueImpacts(dbSession, i));
 187     }
 188
 189     updatedIssues.forEach(i -> issueStorage.insertChanges(changeMapper, i, uuidFactory));
 190   }
 191
 192   private static void persistNoLongerNewIssues(IssueStatistics statistics, List<DefaultIssue> noLongerNewIssues, IssueDao issueDao, DbSession dbSession) {
 193     if (noLongerNewIssues.isEmpty()) {
 194       return;
 195     }
 196
 197     noLongerNewIssues.forEach(i -> {
 198       issueDao.deleteAsNewCodeOnReferenceBranch(dbSession, i.key());
 199       statistics.updates++;
 200     });
 201
 202   }
 203
 204   private void persistNewCodeIssuesToMigrate(IssueStatistics statistics, List<DefaultIssue> newCodeIssuesToMigrate, IssueDao issueDao, DbSession dbSession) {
 205     if (newCodeIssuesToMigrate.isEmpty()) {
 206       return;
 207     }
 208
 209     long now = system2.now();
 210     newCodeIssuesToMigrate.forEach(i -> {
 211       issueDao.insertAsNewCodeOnReferenceBranch(dbSession, NewCodeReferenceIssueDto.fromIssueKey(i.key(), now, uuidFactory));
 212       statistics.updates++;
 213     });
 214   }
 215
 216   private static void flushSession(DbSession dbSession) {
 217     dbSession.flushStatements();
 218     dbSession.commit();
 219   }
 220
 221   private boolean isOnBranchUsingReferenceBranch() {
 222     if (periodHolder.hasPeriod()) {
 223       return periodHolder.getPeriod().getMode().equals(NewCodePeriodType.REFERENCE_BRANCH.name());
 224     }
 225     return false;
 226   }
 227
 228   @Override
 229   public String getDescription() {
 230     return "Persist issues";
 231   }
 232
 233   private static class IssueStatistics {
 234     private int inserts = 0;
 235     private int updates = 0;
 236     private int merged = 0;
 237
 238     private void dumpTo(ComputationStep.Context context) {
 239       context.getStatistics()
 240         .add("inserts", String.valueOf(inserts))
 241         .add("updates", String.valueOf(updates))
 242         .add("merged", String.valueOf(merged));
 243     }
 244   }
 245 }