From acf931ecd25cf442190404ca1064182a7330b603 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Mon, 23 Apr 2018 15:48:38 +0200 Subject: [PATCH] SONAR-10430 do not compute distance for files with too far size --- .../org/sonar/db/component/ComponentDao.java | 16 +- .../sonar/db/component/ComponentMapper.java | 2 + .../sonar/db/component/FileMoveRowDto.java | 59 ++++ .../org/sonar/db/source/FileSourceDao.java | 14 + .../org/sonar/db/source/FileSourceMapper.java | 4 + .../sonar/db/source/LineHashesWithKeyDto.java | 57 ++++ .../sonar/db/component/ComponentMapper.xml | 19 ++ .../org/sonar/db/source/FileSourceMapper.xml | 32 ++- .../ScrollForFileMoveComponentDaoTest.java | 272 ++++++++++++++++++ .../sonar/db/source/FileSourceDaoTest.java | 195 ++++++++++--- ...ProjectAnalysisTaskContainerPopulator.java | 4 +- .../filemove/FileMoveDetectionStep.java | 155 ++++++---- .../filemove/FileSimilarity.java | 6 + .../filemove/MatchesByScore.java | 4 +- .../projectanalysis/filemove/ScoreMatrix.java | 63 ++-- .../filemove/FileMoveDetectionStepTest.java | 49 +++- .../filemove/MatchesByScoreTest.java | 27 +- .../filemove/ScoreMatrixDumperImplTest.java | 8 +- .../filemove/SourceSimilarityImplTest.java | 66 +++-- 19 files changed, 888 insertions(+), 164 deletions(-) create mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/component/FileMoveRowDto.java create mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/source/LineHashesWithKeyDto.java create mode 100644 server/sonar-db-dao/src/test/java/org/sonar/db/component/ScrollForFileMoveComponentDaoTest.java diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentDao.java index c77a5f650f9..e572a95ba15 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentDao.java @@ -334,6 +334,18 @@ public class ComponentDao implements Dao { return mapper(dbSession).selectProjectsByNameQuery(nameQueryForSql, includeModules); } + public List selectComponentKeysHavingIssuesToMerge(DbSession dbSession, String mergeBranchUuid) { + return mapper(dbSession).selectComponentKeysHavingIssuesToMerge(mergeBranchUuid); + } + + /** + * Scroll all enabled files of the specified project (same project_uuid) in no specific order with + * 'SOURCE' source and a non null path. + */ + public void scrollAllFilesForFileMove(DbSession session, String projectUuid, ResultHandler handler) { + mapper(session).scrollAllFilesForFileMove(projectUuid, handler); + } + public void insert(DbSession session, ComponentDto item) { mapper(session).insert(item); } @@ -376,10 +388,6 @@ public class ComponentDao implements Dao { mapper(session).delete(componentId); } - public List selectComponentKeysHavingIssuesToMerge(DbSession dbSession, String mergeBranchUuid) { - return mapper(dbSession).selectComponentKeysHavingIssuesToMerge(mergeBranchUuid); - } - private static void checkThatNotTooManyComponents(ComponentQuery query) { checkThatNotTooManyConditions(query.getComponentIds(), "Too many component ids in query"); checkThatNotTooManyConditions(query.getComponentKeys(), "Too many component keys in query"); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentMapper.java index b33559917e7..eb1023f17a0 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentMapper.java @@ -138,6 +138,8 @@ public interface ComponentMapper { void scrollForIndexing(@Param("projectUuid") @Nullable String projectUuid, ResultHandler handler); + void scrollAllFilesForFileMove(@Param("projectUuid") String projectUuid, ResultHandler handler); + void insert(ComponentDto componentDto); void update(ComponentUpdateDto component); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/FileMoveRowDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/FileMoveRowDto.java new file mode 100644 index 00000000000..5e48dc2596f --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/FileMoveRowDto.java @@ -0,0 +1,59 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.component; + +public class FileMoveRowDto { + private long id; + private String kee; + private String uuid; + private String path; + private int lineCount; + + public long getId() { + return id; + } + + public String getKey() { + return kee; + } + + public String getUuid() { + return uuid; + } + + public String getPath() { + return path; + } + + public int getLineCount() { + return lineCount; + } + + @Override + public String toString() { + return "FileMoveRowDto{" + + "id=" + id + + ", kee='" + kee + '\'' + + ", uuid='" + uuid + '\'' + + ", path='" + path + '\'' + + ", lineCount=" + lineCount + + '}'; + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java index dcdafcbc251..1286e2ed538 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java @@ -25,16 +25,20 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.function.Consumer; import javax.annotation.CheckForNull; import org.apache.commons.dbutils.DbUtils; import org.apache.commons.io.IOUtils; +import org.apache.ibatis.session.ResultHandler; import org.sonar.db.Dao; import org.sonar.db.DbSession; import org.sonar.db.source.FileSourceDto.Type; +import static org.sonar.db.DatabaseUtils.toUniqueAndSortedPartitions; + public class FileSourceDao implements Dao { private static final Splitter END_OF_LINE_SPLITTER = Splitter.on('\n'); @@ -80,6 +84,16 @@ public class FileSourceDao implements Dao { } } + /** + * Scroll line hashes of all enabled components (should be files, but not enforced) with specified + * keys in no specific order with 'SOURCE' source and a non null path. + */ + public void scrollLineHashes(DbSession dbSession, Collection fileKeys, ResultHandler rowHandler) { + for (List partition : toUniqueAndSortedPartitions(fileKeys)) { + mapper(dbSession).scrollLineHashes(partition, rowHandler); + } + } + public void readLineHashesStream(DbSession dbSession, String fileUuid, Consumer consumer) { Connection connection = dbSession.getConnection(); PreparedStatement pstmt = null; diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java index 38deb679be0..3d2f15d6444 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java @@ -19,9 +19,11 @@ */ package org.sonar.db.source; +import java.util.Collection; import java.util.List; import javax.annotation.CheckForNull; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.session.ResultHandler; public interface FileSourceMapper { @@ -30,6 +32,8 @@ public interface FileSourceMapper { @CheckForNull FileSourceDto select(@Param("fileUuid") String fileUuid, @Param("dataType") String dataType); + void scrollLineHashes(@Param("fileKeys") Collection fileKeys, ResultHandler rowHandler); + @CheckForNull Integer selectLineHashesVersion(@Param("fileUuid") String fileUuid, @Param("dataType") String dataType); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/source/LineHashesWithKeyDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/source/LineHashesWithKeyDto.java new file mode 100644 index 00000000000..596c8c7ac08 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/source/LineHashesWithKeyDto.java @@ -0,0 +1,57 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.source; + +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; + +import static org.sonar.db.source.FileSourceDto.LINES_HASHES_SPLITTER; + +public class LineHashesWithKeyDto { + private String kee; + private String path; + private String lineHashes; + + public String getKey() { + return kee; + } + + public String getPath() { + return path; + } + + /** Used by MyBatis */ + public String getRawLineHashes() { + return lineHashes; + } + + /** Used by MyBatis */ + public void setRawLineHashes(@Nullable String lineHashes) { + this.lineHashes = lineHashes; + } + + public List getLineHashes() { + if (lineHashes == null) { + return Collections.emptyList(); + } + return LINES_HASHES_SPLITTER.splitToList(lineHashes); + } +} diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/ComponentMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/ComponentMapper.xml index cac715f12a8..e59d48eefa3 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/ComponentMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/ComponentMapper.xml @@ -496,6 +496,25 @@ + + + +