source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2022 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.ce.task.projectanalysis.filemove;
  21
  22 import java.io.File;
  23 import java.io.IOException;
  24 import java.nio.charset.StandardCharsets;
  25 import java.util.ArrayList;
  26 import java.util.HashMap;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.function.Function;
  30 import java.util.stream.IntStream;
  31 import javax.annotation.CheckForNull;
  32 import javax.annotation.Nullable;
  33 import org.apache.commons.io.FileUtils;
  34 import org.junit.Before;
  35 import org.junit.Rule;
  36 import org.junit.Test;
  37 import org.sonar.api.utils.System2;
  38 import org.sonar.api.utils.log.LogTester;
  39 import org.sonar.api.utils.log.LoggerLevel;
  40 import org.sonar.ce.task.projectanalysis.analysis.Analysis;
  41 import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolderRule;
  42 import org.sonar.ce.task.projectanalysis.component.Component;
  43 import org.sonar.ce.task.projectanalysis.component.FileAttributes;
  44 import org.sonar.ce.task.projectanalysis.component.ReportComponent;
  45 import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule;
  46 import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepository;
  47 import org.sonar.ce.task.step.TestComputationStepContext;
  48 import org.sonar.core.hash.SourceLineHashesComputer;
  49 import org.sonar.core.util.Uuids;
  50 import org.sonar.db.DbClient;
  51 import org.sonar.db.DbTester;
  52 import org.sonar.db.component.ComponentDto;
  53 import org.sonar.db.component.ComponentTesting;
  54 import org.sonar.db.source.FileSourceDto;
  55
  56 import static java.util.Arrays.stream;
  57 import static org.assertj.core.api.Assertions.assertThat;
  58 import static org.mockito.Mockito.mock;
  59 import static org.mockito.Mockito.when;
  60 import static org.sonar.ce.task.projectanalysis.component.ReportComponent.builder;
  61 import static org.sonar.ce.task.projectanalysis.filemove.FileMoveDetectionStep.MIN_REQUIRED_SCORE;
  62
  63 public class FileMoveDetectionStepTest {
  64
  65   private static final String SNAPSHOT_UUID = "uuid_1";
  66   private static final Analysis ANALYSIS = new Analysis.Builder()
  67     .setUuid(SNAPSHOT_UUID)
  68     .setCreatedAt(86521)
  69     .build();
  70   private static final int ROOT_REF = 1;
  71   private static final int FILE_1_REF = 2;
  72   private static final int FILE_2_REF = 3;
  73   private static final int FILE_3_REF = 4;
  74   private static final String[] CONTENT1 = {
  75     "package org.sonar.ce.task.projectanalysis.filemove;",
  76     "",
  77     "public class Foo {",
  78     "  public String bar() {",
  79     "    return \"Doh!\";",
  80     "  }",
  81     "}"
  82   };
  83
  84   private static final String[] LESS_CONTENT1 = {
  85     "package org.sonar.ce.task.projectanalysis.filemove;",
  86     "",
  87     "public class Foo {",
  88     "  public String foo() {",
  89     "    return \"Donut!\";",
  90     "  }",
  91     "}"
  92   };
  93   private static final String[] CONTENT_EMPTY = {
  94     ""
  95   };
  96   private static final String[] CONTENT2 = {
  97     "package org.sonar.ce.queue;",
  98     "",
  99     "import com.google.common.base.MoreObjects;",
 100     "import javax.annotation.CheckForNull;",
 101     "import javax.annotation.Nullable;",
 102     "import javax.annotation.concurrent.Immutable;",
 103     "",
 104     "import static com.google.common.base.Strings.emptyToNull;",
 105     "import static java.util.Objects.requireNonNull;",
 106     "",
 107     "@Immutable",
 108     "public class CeTask {",
 109     "",
 110     ",  private final String type;",
 111     ",  private final String uuid;",
 112     ",  private final String componentUuid;",
 113     ",  private final String componentKey;",
 114     ",  private final String componentName;",
 115     ",  private final String submitterLogin;",
 116     "",
 117     ",  private CeTask(Builder builder) {",
 118     ",    this.uuid = requireNonNull(emptyToNull(builder.uuid));",
 119     ",    this.type = requireNonNull(emptyToNull(builder.type));",
 120     ",    this.componentUuid = emptyToNull(builder.componentUuid);",
 121     ",    this.componentKey = emptyToNull(builder.componentKey);",
 122     ",    this.componentName = emptyToNull(builder.componentName);",
 123     ",    this.submitterLogin = emptyToNull(builder.submitterLogin);",
 124     ",  }",
 125     "",
 126     ",  public String getUuid() {",
 127     ",    return uuid;",
 128     ",  }",
 129     "",
 130     ",  public String getType() {",
 131     ",    return type;",
 132     ",  }",
 133     "",
 134     ",  @CheckForNull",
 135     ",  public String getComponentUuid() {",
 136     ",    return componentUuid;",
 137     ",  }",
 138     "",
 139     ",  @CheckForNull",
 140     ",  public String getComponentKey() {",
 141     ",    return componentKey;",
 142     ",  }",
 143     "",
 144     ",  @CheckForNull",
 145     ",  public String getComponentName() {",
 146     ",    return componentName;",
 147     ",  }",
 148     "",
 149     ",  @CheckForNull",
 150     ",  public String getSubmitterLogin() {",
 151     ",    return submitterLogin;",
 152     ",  }",
 153     ",}",
 154   };
 155   // removed immutable annotation
 156   private static final String[] LESS_CONTENT2 = {
 157     "package org.sonar.ce.queue;",
 158     "",
 159     "import com.google.common.base.MoreObjects;",
 160     "import javax.annotation.CheckForNull;",
 161     "import javax.annotation.Nullable;",
 162     "",
 163     "import static com.google.common.base.Strings.emptyToNull;",
 164     "import static java.util.Objects.requireNonNull;",
 165     "",
 166     "public class CeTask {",
 167     "",
 168     ",  private final String type;",
 169     ",  private final String uuid;",
 170     ",  private final String componentUuid;",
 171     ",  private final String componentKey;",
 172     ",  private final String componentName;",
 173     ",  private final String submitterLogin;",
 174     "",
 175     ",  private CeTask(Builder builder) {",
 176     ",    this.uuid = requireNonNull(emptyToNull(builder.uuid));",
 177     ",    this.type = requireNonNull(emptyToNull(builder.type));",
 178     ",    this.componentUuid = emptyToNull(builder.componentUuid);",
 179     ",    this.componentKey = emptyToNull(builder.componentKey);",
 180     ",    this.componentName = emptyToNull(builder.componentName);",
 181     ",    this.submitterLogin = emptyToNull(builder.submitterLogin);",
 182     ",  }",
 183     "",
 184     ",  public String getUuid() {",
 185     ",    return uuid;",
 186     ",  }",
 187     "",
 188     ",  public String getType() {",
 189     ",    return type;",
 190     ",  }",
 191     "",
 192     ",  @CheckForNull",
 193     ",  public String getComponentUuid() {",
 194     ",    return componentUuid;",
 195     ",  }",
 196     "",
 197     ",  @CheckForNull",
 198     ",  public String getComponentKey() {",
 199     ",    return componentKey;",
 200     ",  }",
 201     "",
 202     ",  @CheckForNull",
 203     ",  public String getComponentName() {",
 204     ",    return componentName;",
 205     ",  }",
 206     "",
 207     ",  @CheckForNull",
 208     ",  public String getSubmitterLogin() {",
 209     ",    return submitterLogin;",
 210     ",  }",
 211     ",}",
 212   };
 213
 214   @Rule
 215   public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule();
 216   @Rule
 217   public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
 218   @Rule
 219   public MutableMovedFilesRepositoryRule movedFilesRepository = new MutableMovedFilesRepositoryRule();
 220   @Rule
 221   public DbTester dbTester = DbTester.create(System2.INSTANCE);
 222   @Rule
 223   public LogTester logTester = new LogTester();
 224
 225   private DbClient dbClient = dbTester.getDbClient();
 226   private ComponentDto project;
 227
 228   private SourceLinesHashRepository sourceLinesHash = mock(SourceLinesHashRepository.class);
 229   private FileSimilarity fileSimilarity = new FileSimilarityImpl(new SourceSimilarityImpl());
 230   private CapturingScoreMatrixDumper scoreMatrixDumper = new CapturingScoreMatrixDumper();
 231   private RecordingMutableAddedFileRepository addedFileRepository = new RecordingMutableAddedFileRepository();
 232
 233   private FileMoveDetectionStep underTest = new FileMoveDetectionStep(analysisMetadataHolder, treeRootHolder, dbClient,
 234     fileSimilarity, movedFilesRepository, sourceLinesHash, scoreMatrixDumper, addedFileRepository);
 235
 236   @Before
 237   public void setUp() throws Exception {
 238     project = dbTester.components().insertPrivateProject();
 239     treeRootHolder.setRoot(builder(Component.Type.PROJECT, ROOT_REF).setUuid(project.uuid()).build());
 240   }
 241
 242   @Test
 243   public void getDescription_returns_description() {
 244     assertThat(underTest.getDescription()).isEqualTo("Detect file moves");
 245   }
 246
 247   @Test
 248   public void execute_detects_no_move_on_first_analysis() {
 249     analysisMetadataHolder.setBaseAnalysis(null);
 250
 251     TestComputationStepContext context = new TestComputationStepContext();
 252     underTest.execute(context);
 253
 254     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 255     verifyStatistics(context, null, null, null, null);
 256   }
 257
 258   @Test
 259   public void execute_detects_no_move_if_baseSnapshot_has_no_file_and_report_has_no_file() {
 260     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 261
 262     TestComputationStepContext context = new TestComputationStepContext();
 263     underTest.execute(context);
 264
 265     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 266     assertThat(addedFileRepository.getComponents()).isEmpty();
 267     verifyStatistics(context, 0, null, null, null);
 268   }
 269
 270   @Test
 271   public void execute_detects_no_move_if_baseSnapshot_has_no_file() {
 272     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 273     Component file1 = fileComponent(FILE_1_REF, null);
 274     Component file2 = fileComponent(FILE_2_REF, null);
 275     setFilesInReport(file1, file2);
 276
 277     TestComputationStepContext context = new TestComputationStepContext();
 278     underTest.execute(context);
 279
 280     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 281     assertThat(addedFileRepository.getComponents()).containsOnly(file1, file2);
 282     verifyStatistics(context, 2, 0, 2, null);
 283   }
 284
 285   @Test
 286   public void execute_detects_no_move_if_there_is_no_file_in_report() {
 287     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 288     insertFiles( /* no components */);
 289     setFilesInReport();
 290
 291     TestComputationStepContext context = new TestComputationStepContext();
 292     underTest.execute(context);
 293
 294     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 295     assertThat(addedFileRepository.getComponents()).isEmpty();
 296     verifyStatistics(context, 0, null, null, null);
 297   }
 298
 299   @Test
 300   public void execute_detects_no_move_if_file_key_exists_in_both_DB_and_report() {
 301     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 302     Component file1 = fileComponent(FILE_1_REF, null);
 303     Component file2 = fileComponent(FILE_2_REF, null);
 304     insertFiles(file1.getUuid(), file2.getUuid());
 305     insertContentOfFileInDb(file1.getUuid(), CONTENT1);
 306     insertContentOfFileInDb(file2.getUuid(), CONTENT2);
 307     setFilesInReport(file2, file1);
 308
 309     TestComputationStepContext context = new TestComputationStepContext();
 310     underTest.execute(context);
 311
 312     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 313     assertThat(addedFileRepository.getComponents()).isEmpty();
 314     verifyStatistics(context, 2, 2, 0, null);
 315   }
 316
 317   @Test
 318   public void execute_detects_move_if_content_of_file_is_same_in_DB_and_report() {
 319     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 320     Component file1 = fileComponent(FILE_1_REF, null);
 321     Component file2 = fileComponent(FILE_2_REF, CONTENT1);
 322     ComponentDto[] dtos = insertFiles(file1.getUuid());
 323     insertContentOfFileInDb(file1.getUuid(), CONTENT1);
 324     setFilesInReport(file2);
 325
 326     TestComputationStepContext context = new TestComputationStepContext();
 327     underTest.execute(context);
 328
 329     assertThat(movedFilesRepository.getComponentsWithOriginal()).containsExactly(file2);
 330     MovedFilesRepository.OriginalFile originalFile = movedFilesRepository.getOriginalFile(file2).get();
 331     assertThat(originalFile.getKey()).isEqualTo(dtos[0].getDbKey());
 332     assertThat(originalFile.getUuid()).isEqualTo(dtos[0].uuid());
 333     assertThat(addedFileRepository.getComponents()).isEmpty();
 334     verifyStatistics(context, 1, 1, 1, 1);
 335   }
 336
 337   @Test
 338   public void execute_detects_no_move_if_content_of_file_is_not_similar_enough() {
 339     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 340     Component file1 = fileComponent(FILE_1_REF, null);
 341     Component file2 = fileComponent(FILE_2_REF, LESS_CONTENT1);
 342     insertFiles(file1.getDbKey());
 343     insertContentOfFileInDb(file1.getDbKey(), CONTENT1);
 344     setFilesInReport(file2);
 345
 346     TestComputationStepContext context = new TestComputationStepContext();
 347     underTest.execute(context);
 348
 349     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 350     assertThat(scoreMatrixDumper.scoreMatrix.getMaxScore())
 351       .isPositive()
 352       .isLessThan(MIN_REQUIRED_SCORE);
 353     assertThat(addedFileRepository.getComponents()).contains(file2);
 354     verifyStatistics(context, 1, 1, 1, 0);
 355   }
 356
 357   @Test
 358   public void execute_detects_no_move_if_content_of_file_is_empty_in_DB() {
 359     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 360     Component file1 = fileComponent(FILE_1_REF, null);
 361     Component file2 = fileComponent(FILE_2_REF, CONTENT1);
 362     insertFiles(file1.getDbKey());
 363     insertContentOfFileInDb(file1.getDbKey(), CONTENT_EMPTY);
 364     setFilesInReport(file2);
 365
 366     TestComputationStepContext context = new TestComputationStepContext();
 367     underTest.execute(context);
 368
 369     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 370     assertThat(scoreMatrixDumper.scoreMatrix.getMaxScore()).isZero();
 371     assertThat(addedFileRepository.getComponents()).contains(file2);
 372     verifyStatistics(context, 1, 1, 1, 0);
 373   }
 374
 375   @Test
 376   public void execute_detects_no_move_if_content_of_file_has_no_path_in_DB() {
 377     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 378     Component file1 = fileComponent(FILE_1_REF, null);
 379     Component file2 = fileComponent(FILE_2_REF, CONTENT1);
 380     insertFiles(key -> newComponentDto(key).setPath(null), file1.getDbKey());
 381     insertContentOfFileInDb(file1.getDbKey(), CONTENT1);
 382     setFilesInReport(file2);
 383
 384     TestComputationStepContext context = new TestComputationStepContext();
 385     underTest.execute(context);
 386
 387     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 388     assertThat(scoreMatrixDumper.scoreMatrix).isNull();
 389     assertThat(addedFileRepository.getComponents()).containsOnly(file2);
 390     verifyStatistics(context, 1, 0, 1, null);
 391   }
 392
 393   @Test
 394   public void execute_detects_no_move_if_content_of_file_is_empty_in_report() {
 395     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 396     Component file1 = fileComponent(FILE_1_REF, null);
 397     Component file2 = fileComponent(FILE_2_REF, CONTENT_EMPTY);
 398     insertFiles(file1.getDbKey());
 399     insertContentOfFileInDb(file1.getDbKey(), CONTENT1);
 400     setFilesInReport(file2);
 401
 402     TestComputationStepContext context = new TestComputationStepContext();
 403     underTest.execute(context);
 404
 405     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 406     assertThat(scoreMatrixDumper.scoreMatrix.getMaxScore()).isZero();
 407     assertThat(addedFileRepository.getComponents()).contains(file2);
 408     verifyStatistics(context, 1, 1, 1, 0);
 409     assertThat(logTester.logs(LoggerLevel.DEBUG)).contains("max score in matrix is less than min required score (85). Do nothing.");
 410   }
 411
 412   @Test
 413   public void execute_detects_no_move_if_two_added_files_have_same_content_as_the_one_in_db() {
 414     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 415     Component file1 = fileComponent(FILE_1_REF, null);
 416     Component file2 = fileComponent(FILE_2_REF, CONTENT1);
 417     Component file3 = fileComponent(FILE_3_REF, CONTENT1);
 418     insertFiles(file1.getDbKey());
 419     insertContentOfFileInDb(file1.getDbKey(), CONTENT1);
 420     setFilesInReport(file2, file3);
 421
 422     TestComputationStepContext context = new TestComputationStepContext();
 423     underTest.execute(context);
 424
 425     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 426     assertThat(scoreMatrixDumper.scoreMatrix.getMaxScore()).isEqualTo(100);
 427     assertThat(addedFileRepository.getComponents()).containsOnly(file2, file3);
 428     verifyStatistics(context, 2, 1, 2, 0);
 429   }
 430
 431   @Test
 432   public void execute_detects_no_move_if_two_deleted_files_have_same_content_as_the_one_added() {
 433     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 434     Component file1 = fileComponent(FILE_1_REF, null);
 435     Component file2 = fileComponent(FILE_2_REF, null);
 436     Component file3 = fileComponent(FILE_3_REF, CONTENT1);
 437     insertFiles(file1.getUuid(), file2.getUuid());
 438     insertContentOfFileInDb(file1.getUuid(), CONTENT1);
 439     insertContentOfFileInDb(file2.getUuid(), CONTENT1);
 440     setFilesInReport(file3);
 441
 442     TestComputationStepContext context = new TestComputationStepContext();
 443     underTest.execute(context);
 444
 445     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 446     assertThat(scoreMatrixDumper.scoreMatrix.getMaxScore()).isEqualTo(100);
 447     assertThat(addedFileRepository.getComponents()).containsOnly(file3);
 448     verifyStatistics(context, 1, 2, 1, 0);
 449   }
 450
 451   @Test
 452   public void execute_detects_no_move_if_two_files_are_empty_in_DB() {
 453     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 454     Component file1 = fileComponent(FILE_1_REF, null);
 455     Component file2 = fileComponent(FILE_2_REF, null);
 456     insertFiles(file1.getUuid(), file2.getUuid());
 457     insertContentOfFileInDb(file1.getUuid(), null);
 458     insertContentOfFileInDb(file2.getUuid(), null);
 459     setFilesInReport(file1, file2);
 460
 461     TestComputationStepContext context = new TestComputationStepContext();
 462     underTest.execute(context);
 463
 464     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 465     assertThat(scoreMatrixDumper.scoreMatrix).isNull();
 466     assertThat(addedFileRepository.getComponents()).isEmpty();
 467     verifyStatistics(context, 2, 2, 0, null);
 468   }
 469
 470   @Test
 471   public void execute_detects_several_moves() {
 472     // testing:
 473     // - file1 renamed to file3
 474     // - file2 deleted
 475     // - file4 untouched
 476     // - file5 renamed to file6 with a small change
 477     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 478     Component file1 = fileComponent(FILE_1_REF, null);
 479     Component file2 = fileComponent(FILE_2_REF, null);
 480     Component file3 = fileComponent(FILE_3_REF, CONTENT1);
 481     Component file4 = fileComponent(5, new String[] {"a", "b"});
 482     Component file5 = fileComponent(6, null);
 483     Component file6 = fileComponent(7, LESS_CONTENT2);
 484     ComponentDto[] dtos = insertFiles(file1.getUuid(), file2.getUuid(), file4.getUuid(), file5.getUuid());
 485     insertContentOfFileInDb(file1.getUuid(), CONTENT1);
 486     insertContentOfFileInDb(file2.getUuid(), LESS_CONTENT1);
 487     insertContentOfFileInDb(file4.getUuid(), new String[] {"e", "f", "g", "h", "i"});
 488     insertContentOfFileInDb(file5.getUuid(), CONTENT2);
 489     setFilesInReport(file3, file4, file6);
 490
 491     TestComputationStepContext context = new TestComputationStepContext();
 492     underTest.execute(context);
 493
 494     assertThat(movedFilesRepository.getComponentsWithOriginal()).containsOnly(file3, file6);
 495     MovedFilesRepository.OriginalFile originalFile2 = movedFilesRepository.getOriginalFile(file3).get();
 496     assertThat(originalFile2.getKey()).isEqualTo(dtos[0].getDbKey());
 497     assertThat(originalFile2.getUuid()).isEqualTo(dtos[0].uuid());
 498     MovedFilesRepository.OriginalFile originalFile5 = movedFilesRepository.getOriginalFile(file6).get();
 499     assertThat(originalFile5.getKey()).isEqualTo(dtos[3].getDbKey());
 500     assertThat(originalFile5.getUuid()).isEqualTo(dtos[3].uuid());
 501     assertThat(scoreMatrixDumper.scoreMatrix.getMaxScore()).isGreaterThan(MIN_REQUIRED_SCORE);
 502     assertThat(addedFileRepository.getComponents()).isEmpty();
 503     verifyStatistics(context, 3, 4, 2, 2);
 504   }
 505
 506   @Test
 507   public void execute_does_not_compute_any_distance_if_all_files_sizes_are_all_too_different() {
 508     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 509     Component file1 = fileComponent(FILE_1_REF, null);
 510     Component file2 = fileComponent(FILE_2_REF, null);
 511     Component file3 = fileComponent(FILE_3_REF, arrayOf(118));
 512     Component file4 = fileComponent(5, arrayOf(25));
 513     insertFiles(file1.getDbKey(), file2.getDbKey());
 514     insertContentOfFileInDb(file1.getDbKey(), arrayOf(100));
 515     insertContentOfFileInDb(file2.getDbKey(), arrayOf(30));
 516     setFilesInReport(file3, file4);
 517
 518     TestComputationStepContext context = new TestComputationStepContext();
 519     underTest.execute(context);
 520
 521     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
 522     assertThat(scoreMatrixDumper.scoreMatrix.getMaxScore()).isZero();
 523     verifyStatistics(context, 2, 2, 2, 0);
 524   }
 525
 526   /**
 527    * Creates an array of {@code numberOfElements} int values as String, starting with zero.
 528    */
 529   private static String[] arrayOf(int numberOfElements) {
 530     return IntStream.range(0, numberOfElements).mapToObj(String::valueOf).toArray(String[]::new);
 531   }
 532
 533   /**
 534    * JH: A bug was encountered in the algorithm and I didn't manage to forge a simpler test case.
 535    */
 536   @Test
 537   public void real_life_use_case() throws Exception {
 538     analysisMetadataHolder.setBaseAnalysis(ANALYSIS);
 539     for (File f : FileUtils.listFiles(new File("src/test/resources/org/sonar/ce/task/projectanalysis/filemove/FileMoveDetectionStepTest/v1"), null, false)) {
 540       insertFiles("uuid_" + f.getName().hashCode());
 541       insertContentOfFileInDb("uuid_" + f.getName().hashCode(), readLines(f));
 542     }
 543
 544     Map<String, Component> comps = new HashMap<>();
 545     int i = 1;
 546     for (File f : FileUtils.listFiles(new File("src/test/resources/org/sonar/ce/task/projectanalysis/filemove/FileMoveDetectionStepTest/v2"), null, false)) {
 547       String[] lines = readLines(f);
 548       Component c = builder(Component.Type.FILE, i++)
 549         .setUuid("uuid_" + f.getName().hashCode())
 550         .setKey(f.getName())
 551         .setName(f.getName())
 552         .setFileAttributes(new FileAttributes(false, null, lines.length))
 553         .build();
 554
 555       comps.put(f.getName(), c);
 556       setFileLineHashesInReport(c, lines);
 557     }
 558
 559     setFilesInReport(comps.values().toArray(new Component[0]));
 560
 561     TestComputationStepContext context = new TestComputationStepContext();
 562     underTest.execute(context);
 563
 564     Component makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex = comps.get("MakeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex.java");
 565     Component migrationRb1238 = comps.get("1238_make_component_uuid_and_analysis_uuid_not_null_on_duplications_index.rb");
 566     Component addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex = comps.get("AddComponentUuidAndAnalysisUuidColumnToDuplicationsIndex.java");
 567     assertThat(movedFilesRepository.getComponentsWithOriginal()).containsOnly(
 568       makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex,
 569       migrationRb1238,
 570       addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex);
 571
 572     assertThat(movedFilesRepository.getOriginalFile(makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex).get().getUuid())
 573       .isEqualTo("uuid_" + "MakeComponentUuidNotNullOnDuplicationsIndex.java".hashCode());
 574     assertThat(movedFilesRepository.getOriginalFile(migrationRb1238).get().getUuid())
 575       .isEqualTo("uuid_" + "1242_make_analysis_uuid_not_null_on_duplications_index.rb".hashCode());
 576     assertThat(movedFilesRepository.getOriginalFile(addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex).get().getUuid())
 577       .isEqualTo("uuid_" + "AddComponentUuidColumnToDuplicationsIndex.java".hashCode());
 578     verifyStatistics(context, comps.values().size(), 12, 6, 3);
 579   }
 580
 581   private String[] readLines(File filename) throws IOException {
 582     return FileUtils
 583       .readLines(filename, StandardCharsets.UTF_8)
 584       .toArray(new String[0]);
 585   }
 586
 587   @CheckForNull
 588   private FileSourceDto insertContentOfFileInDb(String uuid, @Nullable String[] content) {
 589     return dbTester.getDbClient().componentDao().selectByUuid(dbTester.getSession(), uuid)
 590       .map(file -> {
 591         SourceLineHashesComputer linesHashesComputer = new SourceLineHashesComputer();
 592         if (content != null) {
 593           stream(content).forEach(linesHashesComputer::addLine);
 594         }
 595         FileSourceDto fileSourceDto = new FileSourceDto()
 596           .setUuid(Uuids.createFast())
 597           .setFileUuid(file.uuid())
 598           .setProjectUuid(file.projectUuid())
 599           .setLineHashes(linesHashesComputer.getLineHashes());
 600         dbTester.getDbClient().fileSourceDao().insert(dbTester.getSession(), fileSourceDto);
 601         dbTester.commit();
 602         return fileSourceDto;
 603       }).orElse(null);
 604   }
 605
 606   private void setFilesInReport(Component... files) {
 607     treeRootHolder.setRoot(builder(Component.Type.PROJECT, ROOT_REF)
 608       .setUuid(project.uuid())
 609       .addChildren(files)
 610       .build());
 611   }
 612
 613   private ComponentDto[] insertFiles(String... uuids) {
 614     return insertFiles(this::newComponentDto, uuids);
 615   }
 616
 617   private ComponentDto[] insertFiles(Function<String, ComponentDto> newComponentDto, String... uuids) {
 618     return stream(uuids)
 619       .map(newComponentDto)
 620       .map(dto -> dbTester.components().insertComponent(dto))
 621       .toArray(ComponentDto[]::new);
 622   }
 623
 624   private ComponentDto newComponentDto(String uuid) {
 625     return ComponentTesting.newFileDto(project)
 626       .setDbKey("key_" + uuid)
 627       .setUuid(uuid)
 628       .setPath("path_" + uuid);
 629   }
 630
 631   private Component fileComponent(int ref, @Nullable String[] content) {
 632     ReportComponent component = builder(Component.Type.FILE, ref)
 633       .setName("report_path" + ref)
 634       .setFileAttributes(new FileAttributes(false, null, content == null ? 1 : content.length))
 635       .build();
 636     if (content != null) {
 637       setFileLineHashesInReport(component, content);
 638     }
 639     return component;
 640   }
 641
 642   private void setFileLineHashesInReport(Component file, String[] content) {
 643     SourceLineHashesComputer computer = new SourceLineHashesComputer();
 644     for (String line : content) {
 645       computer.addLine(line);
 646     }
 647     when(sourceLinesHash.getLineHashesMatchingDBVersion(file)).thenReturn(computer.getLineHashes());
 648   }
 649
 650   private static class CapturingScoreMatrixDumper implements ScoreMatrixDumper {
 651     private ScoreMatrix scoreMatrix;
 652
 653     @Override
 654     public void dumpAsCsv(ScoreMatrix scoreMatrix) {
 655       this.scoreMatrix = scoreMatrix;
 656     }
 657   }
 658
 659   private static void verifyStatistics(TestComputationStepContext context,
 660     @Nullable Integer expectedReportFiles, @Nullable Integer expectedDbFiles,
 661     @Nullable Integer expectedAddedFiles, @Nullable Integer expectedMovedFiles) {
 662     context.getStatistics().assertValue("reportFiles", expectedReportFiles);
 663     context.getStatistics().assertValue("dbFiles", expectedDbFiles);
 664     context.getStatistics().assertValue("addedFiles", expectedAddedFiles);
 665     context.getStatistics().assertValue("movedFiles", expectedMovedFiles);
 666   }
 667
 668   private static class RecordingMutableAddedFileRepository implements MutableAddedFileRepository {
 669     private final List<Component> components = new ArrayList<>();
 670
 671     @Override
 672     public void register(Component file) {
 673       components.add(file);
 674     }
 675
 676     @Override
 677     public boolean isAdded(Component component) {
 678       throw new UnsupportedOperationException("isAdded should not be called");
 679     }
 680
 681     public List<Component> getComponents() {
 682       return components;
 683     }
 684   }
 685 }