source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2021 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.server.measure.index;
  21
  22 import java.util.Arrays;
  23 import java.util.Collection;
  24 import java.util.function.Consumer;
  25 import java.util.function.Predicate;
  26 import org.elasticsearch.action.search.SearchRequest;
  27 import org.elasticsearch.search.SearchHit;
  28 import org.elasticsearch.search.builder.SearchSourceBuilder;
  29 import org.junit.Rule;
  30 import org.junit.Test;
  31 import org.sonar.api.resources.Qualifiers;
  32 import org.sonar.api.utils.System2;
  33 import org.sonar.db.DbSession;
  34 import org.sonar.db.DbTester;
  35 import org.sonar.db.component.ComponentDto;
  36 import org.sonar.db.component.SnapshotDto;
  37 import org.sonar.db.es.EsQueueDto;
  38 import org.sonar.db.project.ProjectDto;
  39 import org.sonar.server.es.EsTester;
  40 import org.sonar.server.es.IndexingResult;
  41 import org.sonar.server.es.ProjectIndexer;
  42 import org.sonar.server.permission.index.AuthorizationScope;
  43 import org.sonar.server.permission.index.IndexPermissions;
  44
  45 import static java.util.Collections.emptyList;
  46 import static java.util.Collections.emptySet;
  47 import static java.util.Collections.singletonList;
  48 import static org.assertj.core.api.Assertions.assertThat;
  49 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
  50 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
  51 import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
  52 import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto;
  53 import static org.sonar.server.es.EsClient.prepareSearch;
  54 import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
  55 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION;
  56 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION;
  57 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_KEY_UPDATE;
  58 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_TAGS_UPDATE;
  59 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALIFIER;
  60 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
  61 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_UUID;
  62 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
  63 import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
  64
  65 public class ProjectMeasuresIndexerTest {
  66
  67   private System2 system2 = System2.INSTANCE;
  68
  69   @Rule
  70   public EsTester es = EsTester.create();
  71   @Rule
  72   public DbTester db = DbTester.create(system2);
  73
  74   private ProjectMeasuresIndexer underTest = new ProjectMeasuresIndexer(db.getDbClient(), es.client());
  75
  76   @Test
  77   public void test_getAuthorizationScope() {
  78     AuthorizationScope scope = underTest.getAuthorizationScope();
  79     assertThat(scope.getIndexType().getIndex()).isEqualTo(ProjectMeasuresIndexDefinition.DESCRIPTOR);
  80     assertThat(scope.getIndexType().getType()).isEqualTo(TYPE_AUTHORIZATION);
  81
  82     Predicate<IndexPermissions> projectPredicate = scope.getProjectPredicate();
  83     IndexPermissions project = new IndexPermissions("P1", Qualifiers.PROJECT);
  84     IndexPermissions app = new IndexPermissions("P1", Qualifiers.APP);
  85     IndexPermissions file = new IndexPermissions("F1", Qualifiers.FILE);
  86     assertThat(projectPredicate.test(project)).isTrue();
  87     assertThat(projectPredicate.test(app)).isTrue();
  88     assertThat(projectPredicate.test(file)).isFalse();
  89   }
  90
  91   @Test
  92   public void index_nothing() {
  93     underTest.indexOnStartup(emptySet());
  94
  95     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero();
  96   }
  97
  98   @Test
  99   public void indexOnStartup_indexes_all_projects() {
 100     SnapshotDto project1 = db.components().insertProjectAndSnapshot(newPrivateProjectDto());
 101     SnapshotDto project2 = db.components().insertProjectAndSnapshot(newPrivateProjectDto());
 102     SnapshotDto project3 = db.components().insertProjectAndSnapshot(newPrivateProjectDto());
 103
 104     underTest.indexOnStartup(emptySet());
 105
 106     assertThatIndexContainsOnly(project1, project2, project3);
 107     assertThatQualifierIs("TRK", project1, project2, project3);
 108   }
 109
 110   @Test
 111   public void indexAll_indexes_all_projects() {
 112     SnapshotDto project1 = db.components().insertProjectAndSnapshot(newPrivateProjectDto());
 113     SnapshotDto project2 = db.components().insertProjectAndSnapshot(newPrivateProjectDto());
 114     SnapshotDto project3 = db.components().insertProjectAndSnapshot(newPrivateProjectDto());
 115
 116     underTest.indexAll();
 117
 118     assertThatIndexContainsOnly(project1, project2, project3);
 119     assertThatQualifierIs("TRK", project1, project2, project3);
 120   }
 121
 122   /**
 123    * Provisioned projects don't have analysis yet
 124    */
 125   @Test
 126   public void indexOnStartup_indexes_provisioned_projects() {
 127     ComponentDto project = db.components().insertPrivateProject();
 128
 129     underTest.indexOnStartup(emptySet());
 130
 131     assertThatIndexContainsOnly(project);
 132   }
 133
 134   @Test
 135   public void indexOnStartup_ignores_non_main_branches() {
 136     ComponentDto project = db.components().insertPrivateProject();
 137     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setKey("feature/foo"));
 138
 139     underTest.indexOnStartup(emptySet());
 140
 141     assertThatIndexContainsOnly(project);
 142   }
 143
 144   @Test
 145   public void indexOnStartup_indexes_all_applications() {
 146     ComponentDto application1 = db.components().insertPrivateApplication();
 147     ComponentDto application2 = db.components().insertPrivateApplication();
 148     ComponentDto application3 = db.components().insertPrivateApplication();
 149
 150     underTest.indexOnStartup(emptySet());
 151
 152     assertThatIndexContainsOnly(application1, application2, application3);
 153     assertThatQualifierIs("APP", application1, application2, application3);
 154   }
 155
 156   @Test
 157   public void indexOnStartup_indexes_projects_and_applications() {
 158     ComponentDto project1 = db.components().insertPrivateProject();
 159     ComponentDto project2 = db.components().insertPrivateProject();
 160     ComponentDto project3 = db.components().insertPrivateProject();
 161
 162     ComponentDto application1 = db.components().insertPrivateApplication();
 163     ComponentDto application2 = db.components().insertPrivateApplication();
 164     ComponentDto application3 = db.components().insertPrivateApplication();
 165
 166     underTest.indexOnStartup(emptySet());
 167
 168     assertThatIndexContainsOnly(project1, project2, project3, application1, application2, application3);
 169     assertThatQualifierIs("TRK", project1, project2, project3);
 170     assertThatQualifierIs("APP", application1, application2, application3);
 171   }
 172
 173   @Test
 174   public void indexOnAnalysis_indexes_provisioned_project() {
 175     ComponentDto project1 = db.components().insertPrivateProject();
 176     ComponentDto project2 = db.components().insertPrivateProject();
 177
 178     underTest.indexOnAnalysis(project1.uuid());
 179
 180     assertThatIndexContainsOnly(project1);
 181   }
 182
 183   @Test
 184   public void indexOnAnalysis_indexes_provisioned_application() {
 185     ComponentDto app1 = db.components().insertPrivateApplication();
 186     ComponentDto app2 = db.components().insertPrivateApplication();
 187
 188     underTest.indexOnAnalysis(app1.uuid());
 189
 190     assertThatIndexContainsOnly(app1);
 191   }
 192
 193   @Test
 194   public void update_index_when_project_key_is_updated() {
 195     ComponentDto project = db.components().insertPrivateProject();
 196
 197     IndexingResult result = indexProject(project, PROJECT_KEY_UPDATE);
 198
 199     assertThatIndexContainsOnly(project);
 200     assertThat(result.getTotal()).isEqualTo(1L);
 201     assertThat(result.getSuccess()).isEqualTo(1L);
 202   }
 203
 204   @Test
 205   public void update_index_when_project_is_created() {
 206     ComponentDto project = db.components().insertPrivateProject();
 207
 208     IndexingResult result = indexProject(project, PROJECT_CREATION);
 209
 210     assertThatIndexContainsOnly(project);
 211     assertThat(result.getTotal()).isEqualTo(1L);
 212     assertThat(result.getSuccess()).isEqualTo(1L);
 213   }
 214
 215   @Test
 216   public void update_index_when_project_tags_are_updated() {
 217     ComponentDto project = db.components().insertPrivateProject(defaults(), p -> p.setTagsString("foo"));
 218     indexProject(project, PROJECT_CREATION);
 219     assertThatProjectHasTag(project, "foo");
 220
 221     ProjectDto projectDto = db.components().getProjectDto(project);
 222     projectDto.setTagsString("bar");
 223     db.getDbClient().projectDao().updateTags(db.getSession(), projectDto);
 224     // TODO change indexing?
 225     IndexingResult result = indexProject(project, PROJECT_TAGS_UPDATE);
 226
 227     assertThatProjectHasTag(project, "bar");
 228     assertThat(result.getTotal()).isEqualTo(1L);
 229     assertThat(result.getSuccess()).isEqualTo(1L);
 230   }
 231
 232   @Test
 233   public void delete_doc_from_index_when_project_is_deleted() {
 234     ComponentDto project = db.components().insertPrivateProject();
 235     indexProject(project, PROJECT_CREATION);
 236     assertThatIndexContainsOnly(project);
 237
 238     db.getDbClient().purgeDao().deleteProject(db.getSession(), project.uuid());
 239     IndexingResult result = indexProject(project, PROJECT_DELETION);
 240
 241     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero();
 242     assertThat(result.getTotal()).isEqualTo(1L);
 243     assertThat(result.getSuccess()).isEqualTo(1L);
 244   }
 245
 246   @Test
 247   public void do_nothing_if_no_projects_and_apps_to_index() {
 248     // this project should not be indexed
 249     db.components().insertPrivateProject();
 250     db.components().insertPrivateApplication();
 251
 252     underTest.index(db.getSession(), emptyList());
 253
 254     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero();
 255   }
 256
 257   @Test
 258   public void errors_during_indexing_are_recovered() {
 259     ComponentDto project = db.components().insertPrivateProject();
 260     es.lockWrites(TYPE_PROJECT_MEASURES);
 261
 262     IndexingResult result = indexProject(project, PROJECT_CREATION);
 263     assertThat(result.getTotal()).isEqualTo(1L);
 264     assertThat(result.getFailures()).isEqualTo(1L);
 265
 266     // index is still read-only, fail to recover
 267     result = recover();
 268     assertThat(result.getTotal()).isEqualTo(1L);
 269     assertThat(result.getFailures()).isEqualTo(1L);
 270     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero();
 271     assertThatEsQueueTableHasSize(1);
 272
 273     es.unlockWrites(TYPE_PROJECT_MEASURES);
 274
 275     result = recover();
 276     assertThat(result.getTotal()).isEqualTo(1L);
 277     assertThat(result.getFailures()).isEqualTo(0L);
 278     assertThatEsQueueTableHasSize(0);
 279     assertThatIndexContainsOnly(project);
 280   }
 281
 282   @Test
 283   public void non_main_branches_are_not_indexed_during_analysis() {
 284     ComponentDto project = db.components().insertPrivateProject();
 285     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setKey("feature/foo"));
 286
 287     underTest.indexOnAnalysis(branch.uuid());
 288
 289     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero();
 290   }
 291
 292   private IndexingResult indexProject(ComponentDto project, ProjectIndexer.Cause cause) {
 293     DbSession dbSession = db.getSession();
 294     Collection<EsQueueDto> items = underTest.prepareForRecovery(dbSession, singletonList(project.uuid()), cause);
 295     dbSession.commit();
 296     return underTest.index(dbSession, items);
 297   }
 298
 299   private void assertThatProjectHasTag(ComponentDto project, String expectedTag) {
 300     SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
 301       .source(new SearchSourceBuilder()
 302         .query(boolQuery()
 303           .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()))
 304           .filter(termQuery(FIELD_TAGS, expectedTag))));
 305
 306     assertThat(es.client().search(request).getHits().getHits())
 307       .extracting(SearchHit::getId)
 308       .contains(project.uuid());
 309   }
 310
 311   private void assertThatEsQueueTableHasSize(int expectedSize) {
 312     assertThat(db.countRowsOfTable("es_queue")).isEqualTo(expectedSize);
 313   }
 314
 315   private void assertThatIndexContainsOnly(SnapshotDto... expectedProjects) {
 316     assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
 317       Arrays.stream(expectedProjects).map(SnapshotDto::getComponentUuid).toArray(String[]::new));
 318   }
 319
 320   private void assertThatIndexContainsOnly(ComponentDto... expectedProjects) {
 321     assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
 322       Arrays.stream(expectedProjects).map(ComponentDto::uuid).toArray(String[]::new));
 323   }
 324
 325   private void assertThatQualifierIs(String qualifier, ComponentDto... expectedComponents) {
 326     String[] expectedComponentUuids = Arrays.stream(expectedComponents).map(ComponentDto::uuid).toArray(String[]::new);
 327     assertThatQualifierIs(qualifier, expectedComponentUuids);
 328   }
 329
 330   private void assertThatQualifierIs(String qualifier, SnapshotDto... expectedComponents) {
 331     String[] expectedComponentUuids = Arrays.stream(expectedComponents).map(SnapshotDto::getComponentUuid).toArray(String[]::new);
 332     assertThatQualifierIs(qualifier, expectedComponentUuids);
 333   }
 334
 335   private void assertThatQualifierIs(String qualifier, String... componentsUuid) {
 336     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
 337       .query(boolQuery()
 338         .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()))
 339         .filter(termQuery(FIELD_QUALIFIER, qualifier))
 340         .filter(termsQuery(FIELD_UUID, componentsUuid)));
 341
 342     SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
 343       .source(searchSourceBuilder);
 344     assertThat(es.client().search(request).getHits().getHits())
 345       .extracting(SearchHit::getId)
 346       .containsExactlyInAnyOrder(componentsUuid);
 347   }
 348
 349   private IndexingResult recover() {
 350     Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), System.currentTimeMillis() + 1_000L, 10);
 351     return underTest.index(db.getSession(), items);
 352   }
 353
 354   private static <T> Consumer<T> defaults() {
 355     return t -> {
 356       // do nothing
 357     };
 358   }
 359
 360 }