]> source.dussan.org Git - sonarqube.git/blob
3075c6c366a5ad68904d00729777eda11bf5efd3
[sonarqube.git] /
1 /*
2  * SonarQube
3  * Copyright (C) 2009-2016 SonarSource SA
4  * mailto:contact AT sonarsource DOT com
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 3 of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
19  */
20 package org.sonar.server.computation.task.projectanalysis.filemove;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.nio.charset.StandardCharsets;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collections;
28 import java.util.HashMap;
29 import java.util.Iterator;
30 import java.util.List;
31 import java.util.Map;
32 import org.apache.commons.io.FileUtils;
33 import org.junit.Before;
34 import org.junit.Rule;
35 import org.junit.Test;
36 import org.mockito.ArgumentCaptor;
37 import org.sonar.core.hash.SourceHashComputer;
38 import org.sonar.core.hash.SourceLinesHashesComputer;
39 import org.sonar.db.DbClient;
40 import org.sonar.db.DbSession;
41 import org.sonar.db.component.ComponentDao;
42 import org.sonar.db.component.ComponentDto;
43 import org.sonar.db.component.ComponentTreeQuery;
44 import org.sonar.db.source.FileSourceDao;
45 import org.sonar.db.source.FileSourceDto;
46 import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolderRule;
47 import org.sonar.server.computation.task.projectanalysis.component.TreeRootHolderRule;
48 import org.sonar.server.computation.task.projectanalysis.component.Component;
49 import org.sonar.server.computation.task.projectanalysis.component.ReportComponent;
50 import org.sonar.server.computation.task.projectanalysis.analysis.Analysis;
51 import org.sonar.server.computation.task.projectanalysis.source.SourceLinesRepositoryRule;
52
53 import static com.google.common.base.Joiner.on;
54 import static java.util.Arrays.stream;
55 import static java.util.stream.Collectors.toList;
56 import static org.assertj.core.api.Java6Assertions.assertThat;
57 import static org.mockito.Matchers.any;
58 import static org.mockito.Matchers.eq;
59 import static org.mockito.Mockito.mock;
60 import static org.mockito.Mockito.when;
61 import static org.sonar.api.resources.Qualifiers.FILE;
62 import static org.sonar.api.resources.Qualifiers.UNIT_TEST_FILE;
63 import static org.sonar.server.computation.task.projectanalysis.component.ReportComponent.builder;
64
65 public class FileMoveDetectionStepTest {
66
67   private static final long SNAPSHOT_ID = 98765;
68   private static final Analysis ANALYSIS = new Analysis.Builder()
69     .setId(SNAPSHOT_ID)
70     .setUuid("uuid_1")
71     .setCreatedAt(86521)
72     .build();
73   private static final int ROOT_REF = 1;
74   private static final int FILE_1_REF = 2;
75   private static final int FILE_2_REF = 3;
76   private static final int FILE_3_REF = 4;
77   private static final ReportComponent PROJECT = builder(Component.Type.PROJECT, ROOT_REF).build();
78   private static final Component FILE_1 = fileComponent(FILE_1_REF);
79   private static final Component FILE_2 = fileComponent(FILE_2_REF);
80   private static final Component FILE_3 = fileComponent(FILE_3_REF);
81   private static final String[] CONTENT1 = {
82     "package org.sonar.server.computation.task.projectanalysis.filemove;",
83     "",
84     "public class Foo {",
85     "  public String bar() {",
86     "    return \"Doh!\";",
87     "  }",
88     "}"
89   };
90
91   private static final String[] LESS_CONTENT1 = {
92     "package org.sonar.server.computation.task.projectanalysis.filemove;",
93     "",
94     "public class Foo {",
95     "}"
96   };
97   private static final String[] CONTENT_EMPTY = {
98     ""
99   };
100   private static final String[] CONTENT2 = {
101     "package org.sonar.ce.queue;",
102     "",
103     "import com.google.common.base.MoreObjects;",
104     "import javax.annotation.CheckForNull;",
105     "import javax.annotation.Nullable;",
106     "import javax.annotation.concurrent.Immutable;",
107     "",
108     "import static com.google.common.base.Strings.emptyToNull;",
109     "import static java.util.Objects.requireNonNull;",
110     "",
111     "@Immutable",
112     "public class CeTask {",
113     "",
114     ",  private final String type;",
115     ",  private final String uuid;",
116     ",  private final String componentUuid;",
117     ",  private final String componentKey;",
118     ",  private final String componentName;",
119     ",  private final String submitterLogin;",
120     "",
121     ",  private CeTask(Builder builder) {",
122     ",    this.uuid = requireNonNull(emptyToNull(builder.uuid));",
123     ",    this.type = requireNonNull(emptyToNull(builder.type));",
124     ",    this.componentUuid = emptyToNull(builder.componentUuid);",
125     ",    this.componentKey = emptyToNull(builder.componentKey);",
126     ",    this.componentName = emptyToNull(builder.componentName);",
127     ",    this.submitterLogin = emptyToNull(builder.submitterLogin);",
128     ",  }",
129     "",
130     ",  public String getUuid() {",
131     ",    return uuid;",
132     ",  }",
133     "",
134     ",  public String getType() {",
135     ",    return type;",
136     ",  }",
137     "",
138     ",  @CheckForNull",
139     ",  public String getComponentUuid() {",
140     ",    return componentUuid;",
141     ",  }",
142     "",
143     ",  @CheckForNull",
144     ",  public String getComponentKey() {",
145     ",    return componentKey;",
146     ",  }",
147     "",
148     ",  @CheckForNull",
149     ",  public String getComponentName() {",
150     ",    return componentName;",
151     ",  }",
152     "",
153     ",  @CheckForNull",
154     ",  public String getSubmitterLogin() {",
155     ",    return submitterLogin;",
156     ",  }",
157     ",}",
158   };
159   // removed immutable annotation
160   private static final String[] LESS_CONTENT2 = {
161     "package org.sonar.ce.queue;",
162     "",
163     "import com.google.common.base.MoreObjects;",
164     "import javax.annotation.CheckForNull;",
165     "import javax.annotation.Nullable;",
166     "",
167     "import static com.google.common.base.Strings.emptyToNull;",
168     "import static java.util.Objects.requireNonNull;",
169     "",
170     "public class CeTask {",
171     "",
172     ",  private final String type;",
173     ",  private final String uuid;",
174     ",  private final String componentUuid;",
175     ",  private final String componentKey;",
176     ",  private final String componentName;",
177     ",  private final String submitterLogin;",
178     "",
179     ",  private CeTask(Builder builder) {",
180     ",    this.uuid = requireNonNull(emptyToNull(builder.uuid));",
181     ",    this.type = requireNonNull(emptyToNull(builder.type));",
182     ",    this.componentUuid = emptyToNull(builder.componentUuid);",
183     ",    this.componentKey = emptyToNull(builder.componentKey);",
184     ",    this.componentName = emptyToNull(builder.componentName);",
185     ",    this.submitterLogin = emptyToNull(builder.submitterLogin);",
186     ",  }",
187     "",
188     ",  public String getUuid() {",
189     ",    return uuid;",
190     ",  }",
191     "",
192     ",  public String getType() {",
193     ",    return type;",
194     ",  }",
195     "",
196     ",  @CheckForNull",
197     ",  public String getComponentUuid() {",
198     ",    return componentUuid;",
199     ",  }",
200     "",
201     ",  @CheckForNull",
202     ",  public String getComponentKey() {",
203     ",    return componentKey;",
204     ",  }",
205     "",
206     ",  @CheckForNull",
207     ",  public String getComponentName() {",
208     ",    return componentName;",
209     ",  }",
210     "",
211     ",  @CheckForNull",
212     ",  public String getSubmitterLogin() {",
213     ",    return submitterLogin;",
214     ",  }",
215     ",}",
216   };
217
218   @Rule
219   public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule();
220   @Rule
221   public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
222   @Rule
223   public SourceLinesRepositoryRule sourceLinesRepository = new SourceLinesRepositoryRule();
224   @Rule
225   public MutableMovedFilesRepositoryRule movedFilesRepository = new MutableMovedFilesRepositoryRule();
226
227   private DbClient dbClient = mock(DbClient.class);
228   private DbSession dbSession = mock(DbSession.class);
229   private ComponentDao componentDao = mock(ComponentDao.class);
230   private FileSourceDao fileSourceDao = mock(FileSourceDao.class);
231   private FileSimilarity fileSimilarity = new FileSimilarityImpl(new SourceSimilarityImpl());
232   private long dbIdGenerator = 0;
233
234   private FileMoveDetectionStep underTest = new FileMoveDetectionStep(analysisMetadataHolder, treeRootHolder, dbClient,
235     sourceLinesRepository, fileSimilarity, movedFilesRepository);
236
237   @Before
238   public void setUp() throws Exception {
239     when(dbClient.openSession(false)).thenReturn(dbSession);
240     when(dbClient.componentDao()).thenReturn(componentDao);
241     when(dbClient.fileSourceDao()).thenReturn(fileSourceDao);
242     treeRootHolder.setRoot(PROJECT);
243   }
244
245   @Test
246   public void getDescription_returns_description() {
247     assertThat(underTest.getDescription()).isEqualTo("Detect file moves");
248   }
249
250   @Test
251   public void execute_detects_no_move_if_baseProjectSnapshot_is_null() {
252     analysisMetadataHolder.setBaseProjectSnapshot(null);
253
254     underTest.execute();
255
256     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
257   }
258
259   @Test
260   public void execute_detects_no_move_if_baseSnapshot_has_no_file_and_report_has_no_file() {
261     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
262
263     underTest.execute();
264
265     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
266   }
267
268   @Test
269   public void execute_detects_no_move_if_baseSnapshot_has_no_file() {
270     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
271     setFilesInReport(FILE_1, FILE_2);
272
273     underTest.execute();
274
275     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
276   }
277
278   @Test
279   public void execute_retrieves_only_file_and_unit_tests_from_last_snapshot() {
280     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
281     ArgumentCaptor<ComponentTreeQuery> captor = ArgumentCaptor.forClass(ComponentTreeQuery.class);
282     when(componentDao.selectDescendants(eq(dbSession), captor.capture()))
283       .thenReturn(Collections.emptyList());
284
285     underTest.execute();
286
287     ComponentTreeQuery query = captor.getValue();
288     assertThat(query.getBaseUuid()).isEqualTo(PROJECT.getUuid());
289     assertThat(query.getPage()).isEqualTo(1);
290     assertThat(query.getPageSize()).isEqualTo(Integer.MAX_VALUE);
291     assertThat(query.getSqlSort()).isEqualTo("LOWER(p.name) ASC, p.name ASC");
292     assertThat(query.getQualifiers()).containsOnly(FILE, UNIT_TEST_FILE);
293   }
294
295   @Test
296   public void execute_detects_no_move_if_there_is_no_file_in_report() {
297     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
298     mockComponents( /* no components */);
299     setFilesInReport();
300
301     underTest.execute();
302
303     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
304   }
305
306   @Test
307   public void execute_detects_no_move_if_file_key_exists_in_both_DB_and_report() {
308     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
309     mockComponents(FILE_1.getKey(), FILE_2.getKey());
310     setFilesInReport(FILE_2, FILE_1);
311
312     underTest.execute();
313
314     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
315   }
316
317   @Test
318   public void execute_detects_move_if_content_of_file_is_same_in_DB_and_report() {
319     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
320     ComponentDto[] dtos = mockComponents(FILE_1.getKey());
321     mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
322     setFilesInReport(FILE_2);
323     setFileContentInReport(FILE_2_REF, CONTENT1);
324
325     underTest.execute();
326
327     assertThat(movedFilesRepository.getComponentsWithOriginal()).containsExactly(FILE_2);
328     MovedFilesRepository.OriginalFile originalFile = movedFilesRepository.getOriginalFile(FILE_2).get();
329     assertThat(originalFile.getId()).isEqualTo(dtos[0].getId());
330     assertThat(originalFile.getKey()).isEqualTo(dtos[0].getKey());
331     assertThat(originalFile.getUuid()).isEqualTo(dtos[0].uuid());
332   }
333
334   @Test
335   public void execute_detects_no_move_if_content_of_file_is_not_similar_enough() {
336     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
337     mockComponents(FILE_1.getKey());
338     mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
339     setFilesInReport(FILE_2);
340     setFileContentInReport(FILE_2_REF, LESS_CONTENT1);
341
342     underTest.execute();
343
344     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
345   }
346
347   @Test
348   public void execute_detects_no_move_if_content_of_file_is_empty_in_DB() {
349     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
350     mockComponents(FILE_1.getKey());
351     mockContentOfFileInDb(FILE_1.getKey(), CONTENT_EMPTY);
352     setFilesInReport(FILE_2);
353     setFileContentInReport(FILE_2_REF, CONTENT1);
354
355     underTest.execute();
356
357     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
358   }
359
360   @Test
361   public void execute_detects_no_move_if_content_of_file_is_empty_in_report() {
362     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
363     mockComponents(FILE_1.getKey());
364     mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
365     setFilesInReport(FILE_2);
366     setFileContentInReport(FILE_2_REF, CONTENT_EMPTY);
367
368     underTest.execute();
369
370     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
371   }
372
373   @Test
374   public void execute_detects_no_move_if_two_added_files_have_same_content_as_the_one_in_db() {
375     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
376     mockComponents(FILE_1.getKey());
377     mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
378     setFilesInReport(FILE_2, FILE_3);
379     setFileContentInReport(FILE_2_REF, CONTENT1);
380     setFileContentInReport(FILE_3_REF, CONTENT1);
381
382     underTest.execute();
383
384     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
385   }
386
387   @Test
388   public void execute_detects_no_move_if_two_deleted_files_have_same_content_as_the_one_added() {
389     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
390     mockComponents(FILE_1.getKey(), FILE_2.getKey());
391     mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
392     mockContentOfFileInDb(FILE_2.getKey(), CONTENT1);
393     setFilesInReport(FILE_3);
394     setFileContentInReport(FILE_3_REF, CONTENT1);
395
396     underTest.execute();
397
398     assertThat(movedFilesRepository.getComponentsWithOriginal()).isEmpty();
399   }
400
401   @Test
402   public void execute_detects_several_moves() {
403     // testing:
404     // - file1 renamed to file3
405     // - file2 deleted
406     // - file4 untouched
407     // - file5 renamed to file6 with a small change
408     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
409     Component file4 = fileComponent(5);
410     Component file5 = fileComponent(6);
411     Component file6 = fileComponent(7);
412     ComponentDto[] dtos = mockComponents(FILE_1.getKey(), FILE_2.getKey(), file4.getKey(), file5.getKey());
413     mockContentOfFileInDb(FILE_1.getKey(), CONTENT1);
414     mockContentOfFileInDb(FILE_2.getKey(), LESS_CONTENT1);
415     mockContentOfFileInDb(file4.getKey(), new String[] {"e", "f", "g", "h", "i"});
416     mockContentOfFileInDb(file5.getKey(), CONTENT2);
417     setFilesInReport(FILE_3, file4, file6);
418     setFileContentInReport(FILE_3_REF, CONTENT1);
419     setFileContentInReport(file4.getReportAttributes().getRef(), new String[] {"a", "b"});
420     setFileContentInReport(file6.getReportAttributes().getRef(), LESS_CONTENT2);
421
422     underTest.execute();
423
424     assertThat(movedFilesRepository.getComponentsWithOriginal()).containsOnly(FILE_3, file6);
425     MovedFilesRepository.OriginalFile originalFile2 = movedFilesRepository.getOriginalFile(FILE_3).get();
426     assertThat(originalFile2.getId()).isEqualTo(dtos[0].getId());
427     assertThat(originalFile2.getKey()).isEqualTo(dtos[0].getKey());
428     assertThat(originalFile2.getUuid()).isEqualTo(dtos[0].uuid());
429     MovedFilesRepository.OriginalFile originalFile5 = movedFilesRepository.getOriginalFile(file6).get();
430     assertThat(originalFile5.getId()).isEqualTo(dtos[3].getId());
431     assertThat(originalFile5.getKey()).isEqualTo(dtos[3].getKey());
432     assertThat(originalFile5.getUuid()).isEqualTo(dtos[3].uuid());
433   }
434
435   /**
436    * JH: A bug was encountered in the algorithm and I didn't manage to forge a simpler test case.
437    */
438   @Test
439   public void real_life_use_case() throws Exception {
440     analysisMetadataHolder.setBaseProjectSnapshot(ANALYSIS);
441     List<String> componentDtoKey = new ArrayList<>();
442     for (File f : FileUtils.listFiles(new File("src/test/resources/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest/v1"), null, false)) {
443       componentDtoKey.add(f.getName());
444       mockContentOfFileInDb(f.getName(), readLines(f));
445     }
446     mockComponents(componentDtoKey.toArray(new String[0]));
447
448     Map<String, Component> comps = new HashMap<>();
449     int i = 1;
450     for (File f : FileUtils.listFiles(new File("src/test/resources/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest/v2"), null, false)) {
451       comps.put(f.getName(), builder(Component.Type.FILE, i)
452         .setKey(f.getName())
453         .setPath(f.getName())
454         .build());
455       setFileContentInReport(i++, readLines(f));
456     }
457
458     setFilesInReport(comps.values().toArray(new Component[0]));
459
460     underTest.execute();
461
462     Component makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex = comps.get("MakeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex.java");
463     Component migrationRb1238 = comps.get("1238_make_component_uuid_and_analysis_uuid_not_null_on_duplications_index.rb");
464     Component addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex = comps.get("AddComponentUuidAndAnalysisUuidColumnToDuplicationsIndex.java");
465     assertThat(movedFilesRepository.getComponentsWithOriginal()).containsOnly(
466       makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex,
467       migrationRb1238,
468       addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex);
469
470     assertThat(movedFilesRepository.getOriginalFile(makeComponentUuidAndAnalysisUuidNotNullOnDuplicationsIndex).get().getKey())
471       .isEqualTo("MakeComponentUuidNotNullOnDuplicationsIndex.java");
472     assertThat(movedFilesRepository.getOriginalFile(migrationRb1238).get().getKey())
473       .isEqualTo("1242_make_analysis_uuid_not_null_on_duplications_index.rb");
474     assertThat(movedFilesRepository.getOriginalFile(addComponentUuidAndAnalysisUuidColumnToDuplicationsIndex).get().getKey())
475       .isEqualTo("AddComponentUuidColumnToDuplicationsIndex.java");
476   }
477
478   private String[] readLines(File filename) throws IOException {
479     return FileUtils
480       .readLines(filename, StandardCharsets.UTF_8)
481       .toArray(new String[0]);
482   }
483
484   private void setFileContentInReport(int ref, String[] content) {
485     sourceLinesRepository.addLines(ref, content);
486   }
487
488   private void mockContentOfFileInDb(String key, String[] content) {
489     SourceLinesHashesComputer linesHashesComputer = new SourceLinesHashesComputer();
490     SourceHashComputer sourceHashComputer = new SourceHashComputer();
491     Iterator<String> lineIterator = Arrays.asList(content).iterator();
492     while (lineIterator.hasNext()) {
493       String line = lineIterator.next();
494       linesHashesComputer.addLine(line);
495       sourceHashComputer.addLine(line, lineIterator.hasNext());
496     }
497
498     when(fileSourceDao.selectSourceByFileUuid(dbSession, componentUuidOf(key)))
499       .thenReturn(new FileSourceDto()
500         .setLineHashes(on('\n').join(linesHashesComputer.getLineHashes()))
501         .setSrcHash(sourceHashComputer.getHash()));
502   }
503
504   private void setFilesInReport(Component... files) {
505     treeRootHolder.setRoot(builder(Component.Type.PROJECT, ROOT_REF)
506       .addChildren(files)
507       .build());
508   }
509
510   private ComponentDto[] mockComponents(String... componentKeys) {
511     List<ComponentDto> componentDtos = stream(componentKeys)
512       .map(key -> newComponentDto(key))
513       .collect(toList());
514     when(componentDao.selectDescendants(eq(dbSession), any(ComponentTreeQuery.class)))
515       .thenReturn(componentDtos);
516     return componentDtos.toArray(new ComponentDto[componentDtos.size()]);
517   }
518
519   private ComponentDto newComponentDto(String key) {
520     ComponentDto res = new ComponentDto();
521     res
522       .setId(dbIdGenerator)
523       .setKey(key)
524       .setUuid(componentUuidOf(key))
525       .setPath("path_" + key);
526     dbIdGenerator++;
527     return res;
528   }
529
530   private static String componentUuidOf(String key) {
531     return "uuid_" + key;
532   }
533
534   private static Component fileComponent(int ref) {
535     return builder(Component.Type.FILE, ref)
536       .setPath("report_path" + ref)
537       .build();
538   }
539
540 }