3 * Copyright (C) 2009-2016 SonarSource SA
4 * mailto:contact AT sonarsource DOT com
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 3 of the License, or (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 package org.sonar.server.computation.task.projectanalysis.filemove;
23 import java.io.IOException;
24 import java.nio.charset.StandardCharsets;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collections;
28 import java.util.HashMap;
29 import java.util.Iterator;
30 import java.util.List;
32 import org.apache.commons.io.FileUtils;
33 import org.junit.Before;
34 import org.junit.Rule;
35 import org.junit.Test;
36 import org.mockito.ArgumentCaptor;
37 import org.sonar.core.hash.SourceHashComputer;
38 import org.sonar.core.hash.SourceLinesHashesComputer;
39 import org.sonar.db.DbClient;
40 import org.sonar.db.DbSession;
41 import org.sonar.db.component.ComponentDao;
42 import org.sonar.db.component.ComponentDto;
43 import org.sonar.db.component.ComponentTreeQuery;
44 import org.sonar.db.source.FileSourceDao;
45 import org.sonar.db.source.FileSourceDto;
46 import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolderRule;
47 import org.sonar.server.computation.task.projectanalysis.component.TreeRootHolderRule;
48 import org.sonar.server.computation.task.projectanalysis.component.Component;
49 import org.sonar.server.computation.task.projectanalysis.component.ReportComponent;
50 import org.sonar.server.computation.task.projectanalysis.analysis.Analysis;
51 import org.sonar.server.computation.task.projectanalysis.source.SourceLinesRepositoryRule;
53 import static com.google.common.base.Joiner.on;
54 import static java.util.Arrays.stream;
55 import static java.util.stream.Collectors.toList;
56 import static org.assertj.core.api.Java6Assertions.assertThat;
57 import static org.mockito.Matchers.any;
58 import static org.mockito.Matchers.eq;
59 import static org.mockito.Mockito.mock;
60 import static org.mockito.Mockito.when;
61 import static org.sonar.api.resources.Qualifiers.FILE;
62 import static org.sonar.api.resources.Qualifiers.UNIT_TEST_FILE;
63 import static org.sonar.server.computation.task.projectanalysis.component.ReportComponent.builder;
65 public class FileMoveDetectionStepTest {
67 private static final long SNAPSHOT_ID = 98765;
68 private static final Analysis ANALYSIS = new Analysis.Builder()
73 private static final int ROOT_REF = 1;
74 private static final int FILE_1_REF = 2;
75 private static final int FILE_2_REF = 3;
76 private static final int FILE_3_REF = 4;
77 private static final ReportComponent PROJECT = builder(Component.Type.PROJECT, ROOT_REF).build();
78 private static final Component FILE_1 = fileComponent(FILE_1_REF);
79 private static final Component FILE_2 = fileComponent(FILE_2_REF);
80 private static final Component FILE_3 = fileComponent(FILE_3_REF);
81 private static final String[] CONTENT1 = {
82 "package org.sonar.server.computation.task.projectanalysis.filemove;",
85 " public String bar() {",
91 private static final String[] LESS_CONTENT1 = {
92 "package org.sonar.server.computation.task.projectanalysis.filemove;",
97 private static final String[] CONTENT_EMPTY = {
100 private static final String[] CONTENT2 = {
101 "package org.sonar.ce.queue;",
103 "import com.google.common.base.MoreObjects;",
104 "import javax.annotation.CheckForNull;",
105 "import javax.annotation.Nullable;",
106 "import javax.annotation.concurrent.Immutable;",
108 "import static com.google.common.base.Strings.emptyToNull;",
109 "import static java.util.Objects.requireNonNull;",
112 "public class CeTask {",
114 ", private final String type;",
115 ", private final String uuid;",
116 ", private final String componentUuid;",
117 ", private final String componentKey;",
118 ", private final String componentName;",
119 ", private final String submitterLogin;",
121 ", private CeTask(Builder builder) {",
122 ", this.uuid = requireNonNull(emptyToNull(builder.uuid));",
123 ", this.type = requireNonNull(emptyToNull(builder.type));",
124 ", this.componentUuid = emptyToNull(builder.componentUuid);",
125 ", this.componentKey = emptyToNull(builder.componentKey);",
126 ", this.componentName = emptyToNull(builder.componentName);",
127 ", this.submitterLogin = emptyToNull(builder.submitterLogin);",
130 ", public String getUuid() {",
134 ", public String getType() {",
139 ", public String getComponentUuid() {",
140 ", return componentUuid;",
144 ", public String getComponentKey() {",
145 ", return componentKey;",
149 ", public String getComponentName() {",
150 ", return componentName;",
154 ", public String getSubmitterLogin() {",
155 ", return submitterLogin;",
159 // removed immutable annotation
160 private static final String[] LESS_CONTENT2 = {
161 "package org.sonar.ce.queue;",
163 "import com.google.common.base.MoreObjects;",
164 "import javax.annotation.CheckForNull;",
165 "import javax.annotation.Nullable;",
167 "import static com.google.common.base.Strings.emptyToNull;",
168 "import static java.util.Objects.requireNonNull;",
170 "public class CeTask {",
172 ", private final String type;",
173 ", private final String uuid;",
174 ", private final String componentUuid;",
175 ", private final String componentKey;",
176 ", private final String componentName;",
177 ", private final String submitterLogin;",
179 ", private CeTask(Builder builder) {",
180 ", this.uuid = requireNonNull(emptyToNull(builder.uuid));",
181 ", this.type = requireNonNull(emptyToNull(builder.type));",
182 ", this.componentUuid = emptyToNull(builder.componentUuid);",
183 ", this.componentKey = emptyToNull(builder.componentKey);",
184 ", this.componentName = emptyToNull(builder.componentName);",
185 ", this.submitterLogin = emptyToNull(builder.submitterLogin);",
188 ", public String getUuid() {",
192 ", public String getType() {",
197 ", public String getComponentUuid() {",
198 ", return componentUuid;",
202 ", public String getComponentKey() {",
203 ", return componentKey;",
207 ", public String getComponentName() {",
208 ", return componentName;",
212 ", public String getSubmitterLogin() {",
213 ", return submitterLogin;",
219 public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule();
221 public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
223 public SourceLinesRepositoryRule sourceLinesRepository = new SourceLinesRepositoryRule();
225 public MutableMovedFilesRepositoryRule movedFilesRepository = new MutableMovedFilesRepositoryRule();
227 private DbClient dbClient = mock(DbClient.class);
228 private DbSession dbSession = mock(DbSession.class);
229 private ComponentDao componentDao = mock(ComponentDao.class);
230 private FileSourceDao fileSourceDao = mock(FileSourceDao.class);
231 private FileSimilarity fileSimilarity = new FileSimilarityImpl(new SourceSimilarityImpl());
232 private long dbIdGenerator = 0;
234 private FileMoveDetectionStep underTest = new FileMoveDetectionStep(analysisMetadataHolder, treeRootHolder, dbClient,
235 sourceLinesRepository, fileSimilarity, movedFilesRepository);
238 public void setUp() throws Exception {
239 when(dbClient.openSession(false)).thenReturn(dbSession);
240 when(dbClient.componentDao()).thenReturn(componentDao);
241 when(dbClient.fileSourceDao()).thenReturn(fileSourceDao);
242 treeRootHolder.setRoot(PROJECT);
246 public void getDescription_returns_description() {
247 assertThat(underTest.getDescription()).isEqualTo("Detect file moves");
251 public void execute_detects_no_move_if_baseProjectSnapshot_is_null() {
252 analysisMetadataHolder.setBaseProjectSnapshot(null);
256 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
260 public void execute_detects_no_move_if_baseSnapshot_has_no_file_and_report_has_no_file() {
261 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
265 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
269 public void execute_detects_no_move_if_baseSnapshot_has_no_file() {
270 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
271 setFilesInReport(FILE_1, FILE_2);
275 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
279 public void execute_retrieves_only_file_and_unit_tests_from_last_snapshot() {
280 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
281 ArgumentCaptor<ComponentTreeQuery> captor = ArgumentCaptor.forClass(ComponentTreeQuery.class);
282 when(componentDao.selectDescendants(eq(dbSession), captor.capture()))
283 .thenReturn(Collections.emptyList());
287 ComponentTreeQuery query = captor.getValue();
288 assertThat(query.getBaseUuid()).isEqualTo(PROJECT.getUuid());
289 assertThat(query.getPage()).isEqualTo(1);
290 assertThat(query.getPageSize()).isEqualTo(Integer.MAX_VALUE);
291 assertThat(query.getSqlSort()).isEqualTo("LOWER(p.name) ASC, p.name ASC");
292 assertThat(query.getQualifiers()).containsOnly(FILE, UNIT_TEST_FILE);
296 public void execute_detects_no_move_if_there_is_no_file_in_report() {
297 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
298 mockComponents( /* no components */);
303 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
307 public void execute_detects_no_move_if_file_key_exists_in_both_DB_and_report() {
308 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
309 mockComponents(FILE_1.getKey(), FILE_2.getKey());
310 setFilesInReport(FILE_2, FILE_1);
314 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
318 public void execute_detects_move_if_content_of_file_is_same_in_DB_and_report() {
319 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
320 ComponentDto[] dtos = mockComponents(FILE_1.getKey());
321 mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
322 setFilesInReport(FILE_2);
323 setFileContentInReport(FILE_2_REF, CONTENT1);
327 assertThat(movedFilesRepository.getComponentsWithOriginal()).containsExactly(FILE_2);
328 MovedFilesRepository.OriginalFile originalFile = movedFilesRepository.getOriginalFile(FILE_2).get();
329 assertThat(originalFile.getId()).isEqualTo(dtos[0].getId());
330 assertThat(originalFile.getKey()).isEqualTo(dtos[0].getKey());
331 assertThat(originalFile.getUuid()).isEqualTo(dtos[0].uuid());
335 public void execute_detects_no_move_if_content_of_file_is_not_similar_enough() {
336 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
337 mockComponents(FILE_1.getKey());
338 mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
339 setFilesInReport(FILE_2);
340 setFileContentInReport(FILE_2_REF, LESS_CONTENT1);
344 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
348 public void execute_detects_no_move_if_content_of_file_is_empty_in_DB() {
349 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
350 mockComponents(FILE_1.getKey());
351 mockContentOfFileInDb(FILE_1.getKey(), CONTENT_EMPTY);
352 setFilesInReport(FILE_2);
353 setFileContentInReport(FILE_2_REF, CONTENT1);
357 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
361 public void execute_detects_no_move_if_content_of_file_is_empty_in_report() {
362 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
363 mockComponents(FILE_1.getKey());
364 mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
365 setFilesInReport(FILE_2);
366 setFileContentInReport(FILE_2_REF, CONTENT_EMPTY);
370 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
374 public void execute_detects_no_move_if_two_added_files_have_same_content_as_the_one_in_db() {
375 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
376 mockComponents(FILE_1.getKey());
377 mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
378 setFilesInReport(FILE_2, FILE_3);
379 setFileContentInReport(FILE_2_REF, CONTENT1);
380 setFileContentInReport(FILE_3_REF, CONTENT1);
384 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
388 public void execute_detects_no_move_if_two_deleted_files_have_same_content_as_the_one_added() {
389 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
390 mockComponents(FILE_1.getKey(), FILE_2.getKey());
391 mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
392 mockContentOfFileInDb(FILE_2.getKey(), CONTENT1);
393 setFilesInReport(FILE_3);
394 setFileContentInReport(FILE_3_REF, CONTENT1);
398 assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
402 public void execute_detects_several_moves() {
404 // - file1 renamed to file3
407 // - file5 renamed to file6 with a small change
408 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
409 Component file4 = fileComponent(5);
410 Component file5 = fileComponent(6);
411 Component file6 = fileComponent(7);
412 ComponentDto[] dtos = mockComponents(FILE_1.getKey(), FILE_2.getKey(), file4.getKey(), file5.getKey());
413 mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
414 mockContentOfFileInDb(FILE_2.getKey(), LESS_CONTENT1);
415 mockContentOfFileInDb(file4.getKey(), new String[] {"e", "f", "g", "h", "i"});
416 mockContentOfFileInDb(file5.getKey(), CONTENT2);
417 setFilesInReport(FILE_3, file4, file6);
418 setFileContentInReport(FILE_3_REF, CONTENT1);
419 setFileContentInReport(file4.getReportAttributes().getRef(), new String[] {"a", "b"});
420 setFileContentInReport(file6.getReportAttributes().getRef(), LESS_CONTENT2);
424 assertThat(movedFilesRepository.getComponentsWithOriginal()).containsOnly(FILE_3, file6);
425 MovedFilesRepository.OriginalFile originalFile2 = movedFilesRepository.getOriginalFile(FILE_3).get();
426 assertThat(originalFile2.getId()).isEqualTo(dtos[0].getId());
427 assertThat(originalFile2.getKey()).isEqualTo(dtos[0].getKey());
428 assertThat(originalFile2.getUuid()).isEqualTo(dtos[0].uuid());
429 MovedFilesRepository.OriginalFile originalFile5 = movedFilesRepository.getOriginalFile(file6).get();
430 assertThat(originalFile5.getId()).isEqualTo(dtos[3].getId());
431 assertThat(originalFile5.getKey()).isEqualTo(dtos[3].getKey());
432 assertThat(originalFile5.getUuid()).isEqualTo(dtos[3].uuid());
436 * JH: A bug was encountered in the algorithm and I didn't manage to forge a simpler test case.
439 public void real_life_use_case() throws Exception {
440 analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
441 List<String> componentDtoKey = new ArrayList<>();
442 for (File f : FileUtils.listFiles(new File("src/test/resources/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest/v1"), null, false)) {
443 componentDtoKey.add(f.getName());
444 mockContentOfFileInDb(f.getName(), readLines(f));
446 mockComponents(componentDtoKey.toArray(new String[0]));
448 Map<String, Component> comps = new HashMap<>();
450 for (File f : FileUtils.listFiles(new File("src/test/resources/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest/v2"), null, false)) {
451 comps.put(f.getName(), builder(Component.Type.FILE, i)
453 .setPath(f.getName())
455 setFileContentInReport(i++, readLines(f));
458 setFilesInReport(comps.values().toArray(new Component[0]));
462 Component makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex = comps.get("MakeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex.java");
463 Component migrationRb1238 = comps.get("1238_make_component_uuid_and_analysis_uuid_not_null_on_duplications_index.rb");
464 Component addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex = comps.get("AddComponentUuidAndAnalysisUuidColumnToDuplicationsIndex.java");
465 assertThat(movedFilesRepository.getComponentsWithOriginal()).containsOnly(
466 makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex,
468 addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex);
470 assertThat(movedFilesRepository.getOriginalFile(makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex).get().getKey())
471 .isEqualTo("MakeComponentUuidNotNullOnDuplicationsIndex.java");
472 assertThat(movedFilesRepository.getOriginalFile(migrationRb1238).get().getKey())
473 .isEqualTo("1242_make_analysis_uuid_not_null_on_duplications_index.rb");
474 assertThat(movedFilesRepository.getOriginalFile(addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex).get().getKey())
475 .isEqualTo("AddComponentUuidColumnToDuplicationsIndex.java");
478 private String[] readLines(File filename) throws IOException {
480 .readLines(filename, StandardCharsets.UTF_8)
481 .toArray(new String[0]);
484 private void setFileContentInReport(int ref, String[] content) {
485 sourceLinesRepository.addLines(ref, content);
488 private void mockContentOfFileInDb(String key, String[] content) {
489 SourceLinesHashesComputer linesHashesComputer = new SourceLinesHashesComputer();
490 SourceHashComputer sourceHashComputer = new SourceHashComputer();
491 Iterator<String> lineIterator = Arrays.asList(content).iterator();
492 while (lineIterator.hasNext()) {
493 String line = lineIterator.next();
494 linesHashesComputer.addLine(line);
495 sourceHashComputer.addLine(line, lineIterator.hasNext());
498 when(fileSourceDao.selectSourceByFileUuid(dbSession, componentUuidOf(key)))
499 .thenReturn(new FileSourceDto()
500 .setLineHashes(on('\n').join(linesHashesComputer.getLineHashes()))
501 .setSrcHash(sourceHashComputer.getHash()));
504 private void setFilesInReport(Component... files) {
505 treeRootHolder.setRoot(builder(Component.Type.PROJECT, ROOT_REF)
510 private ComponentDto[] mockComponents(String... componentKeys) {
511 List<ComponentDto> componentDtos = stream(componentKeys)
512 .map(key -> newComponentDto(key))
514 when(componentDao.selectDescendants(eq(dbSession), any(ComponentTreeQuery.class)))
515 .thenReturn(componentDtos);
516 return componentDtos.toArray(new ComponentDto[componentDtos.size()]);
519 private ComponentDto newComponentDto(String key) {
520 ComponentDto res = new ComponentDto();
522 .setId(dbIdGenerator)
524 .setUuid(componentUuidOf(key))
525 .setPath("path_" + key);
530 private static String componentUuidOf(String key) {
531 return "uuid_" + key;
534 private static Component fileComponent(int ref) {
535 return builder(Component.Type.FILE, ref)
536 .setPath("report_path" + ref)