source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2020 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.server.measure.index;
  21
  22 import java.util.Arrays;
  23 import java.util.Collection;
  24 import java.util.function.Consumer;
  25 import java.util.function.Predicate;
  26 import org.elasticsearch.action.search.SearchRequest;
  27 import org.elasticsearch.search.SearchHit;
  28 import org.elasticsearch.search.builder.SearchSourceBuilder;
  29 import org.junit.Rule;
  30 import org.junit.Test;
  31 import org.sonar.api.resources.Qualifiers;
  32 import org.sonar.api.utils.System2;
  33 import org.sonar.db.DbSession;
  34 import org.sonar.db.DbTester;
  35 import org.sonar.db.component.ComponentDto;
  36 import org.sonar.db.component.SnapshotDto;
  37 import org.sonar.db.es.EsQueueDto;
  38 import org.sonar.db.organization.OrganizationDto;
  39 import org.sonar.db.project.ProjectDto;
  40 import org.sonar.server.es.EsTester;
  41 import org.sonar.server.es.IndexingResult;
  42 import org.sonar.server.es.ProjectIndexer;
  43 import org.sonar.server.permission.index.AuthorizationScope;
  44 import org.sonar.server.permission.index.IndexPermissions;
  45
  46 import static java.util.Collections.emptyList;
  47 import static java.util.Collections.emptySet;
  48 import static java.util.Collections.singletonList;
  49 import static org.assertj.core.api.Assertions.assertThat;
  50 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
  51 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
  52 import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
  53 import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto;
  54 import static org.sonar.server.es.EsClient.prepareSearch;
  55 import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
  56 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION;
  57 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION;
  58 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_KEY_UPDATE;
  59 import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_TAGS_UPDATE;
  60 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALIFIER;
  61 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
  62 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_UUID;
  63 import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
  64 import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
  65
  66 public class ProjectMeasuresIndexerTest {
  67
  68   private System2 system2 = System2.INSTANCE;
  69
  70   @Rule
  71   public EsTester es = EsTester.create();
  72   @Rule
  73   public DbTester db = DbTester.create(system2);
  74
  75   private ProjectMeasuresIndexer underTest = new ProjectMeasuresIndexer(db.getDbClient(), es.client());
  76
  77   @Test
  78   public void test_getAuthorizationScope() {
  79     AuthorizationScope scope = underTest.getAuthorizationScope();
  80     assertThat(scope.getIndexType().getIndex()).isEqualTo(ProjectMeasuresIndexDefinition.DESCRIPTOR);
  81     assertThat(scope.getIndexType().getType()).isEqualTo(TYPE_AUTHORIZATION);
  82
  83     Predicate<IndexPermissions> projectPredicate = scope.getProjectPredicate();
  84     IndexPermissions project = new IndexPermissions("P1", Qualifiers.PROJECT);
  85     IndexPermissions app = new IndexPermissions("P1", Qualifiers.APP);
  86     IndexPermissions file = new IndexPermissions("F1", Qualifiers.FILE);
  87     assertThat(projectPredicate.test(project)).isTrue();
  88     assertThat(projectPredicate.test(app)).isTrue();
  89     assertThat(projectPredicate.test(file)).isFalse();
  90   }
  91
  92   @Test
  93   public void index_nothing() {
  94     underTest.indexOnStartup(emptySet());
  95
  96     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero();
  97   }
  98
  99   @Test
 100   public void indexOnStartup_indexes_all_projects() {
 101     OrganizationDto organization = db.organizations().insert();
 102     SnapshotDto project1 = db.components().insertProjectAndSnapshot(newPrivateProjectDto(organization));
 103     SnapshotDto project2 = db.components().insertProjectAndSnapshot(newPrivateProjectDto(organization));
 104     SnapshotDto project3 = db.components().insertProjectAndSnapshot(newPrivateProjectDto(organization));
 105
 106     underTest.indexOnStartup(emptySet());
 107
 108     assertThatIndexContainsOnly(project1, project2, project3);
 109     assertThatQualifierIs("TRK", project1, project2, project3);
 110   }
 111
 112   /**
 113    * Provisioned projects don't have analysis yet
 114    */
 115   @Test
 116   public void indexOnStartup_indexes_provisioned_projects() {
 117     ComponentDto project = db.components().insertPrivateProject();
 118
 119     underTest.indexOnStartup(emptySet());
 120
 121     assertThatIndexContainsOnly(project);
 122   }
 123
 124   @Test
 125   public void indexOnStartup_ignores_non_main_branches() {
 126     ComponentDto project = db.components().insertPrivateProject();
 127     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setKey("feature/foo"));
 128
 129     underTest.indexOnStartup(emptySet());
 130
 131     assertThatIndexContainsOnly(project);
 132   }
 133
 134   @Test
 135   public void indexOnStartup_indexes_all_applications() {
 136     OrganizationDto organization = db.organizations().insert();
 137     ComponentDto application1 = db.components().insertPrivateApplication(organization);
 138     ComponentDto application2 = db.components().insertPrivateApplication(organization);
 139     ComponentDto application3 = db.components().insertPrivateApplication(organization);
 140
 141     underTest.indexOnStartup(emptySet());
 142
 143     assertThatIndexContainsOnly(application1, application2, application3);
 144     assertThatQualifierIs("APP", application1, application2, application3);
 145   }
 146
 147   @Test
 148   public void indexOnStartup_indexes_projects_and_applications() {
 149     OrganizationDto organization = db.organizations().insert();
 150
 151     ComponentDto project1 = db.components().insertPrivateProject();
 152     ComponentDto project2 = db.components().insertPrivateProject();
 153     ComponentDto project3 = db.components().insertPrivateProject();
 154
 155     ComponentDto application1 = db.components().insertPrivateApplication(organization);
 156     ComponentDto application2 = db.components().insertPrivateApplication(organization);
 157     ComponentDto application3 = db.components().insertPrivateApplication(organization);
 158
 159     underTest.indexOnStartup(emptySet());
 160
 161     assertThatIndexContainsOnly(project1, project2, project3, application1, application2, application3);
 162     assertThatQualifierIs("TRK", project1, project2, project3);
 163     assertThatQualifierIs("APP", application1, application2, application3);
 164   }
 165
 166   @Test
 167   public void indexOnAnalysis_indexes_provisioned_project() {
 168     ComponentDto project1 = db.components().insertPrivateProject();
 169     ComponentDto project2 = db.components().insertPrivateProject();
 170
 171     underTest.indexOnAnalysis(project1.uuid());
 172
 173     assertThatIndexContainsOnly(project1);
 174   }
 175
 176   @Test
 177   public void indexOnAnalysis_indexes_provisioned_application() {
 178     ComponentDto app1 = db.components().insertPrivateApplication();
 179     ComponentDto app2 = db.components().insertPrivateApplication();
 180
 181     underTest.indexOnAnalysis(app1.uuid());
 182
 183     assertThatIndexContainsOnly(app1);
 184   }
 185
 186   @Test
 187   public void update_index_when_project_key_is_updated() {
 188     ComponentDto project = db.components().insertPrivateProject();
 189
 190     IndexingResult result = indexProject(project, PROJECT_KEY_UPDATE);
 191
 192     assertThatIndexContainsOnly(project);
 193     assertThat(result.getTotal()).isEqualTo(1L);
 194     assertThat(result.getSuccess()).isEqualTo(1L);
 195   }
 196
 197   @Test
 198   public void update_index_when_project_is_created() {
 199     ComponentDto project = db.components().insertPrivateProject();
 200
 201     IndexingResult result = indexProject(project, PROJECT_CREATION);
 202
 203     assertThatIndexContainsOnly(project);
 204     assertThat(result.getTotal()).isEqualTo(1L);
 205     assertThat(result.getSuccess()).isEqualTo(1L);
 206   }
 207
 208   @Test
 209   public void update_index_when_project_tags_are_updated() {
 210     ComponentDto project = db.components().insertPrivateProject(defaults(), p -> p.setTagsString("foo"));
 211     indexProject(project, PROJECT_CREATION);
 212     assertThatProjectHasTag(project, "foo");
 213
 214     ProjectDto projectDto = db.components().getProjectDto(project);
 215     projectDto.setTagsString("bar");
 216     db.getDbClient().projectDao().updateTags(db.getSession(), projectDto);
 217     // TODO change indexing?
 218     IndexingResult result = indexProject(project, PROJECT_TAGS_UPDATE);
 219
 220     assertThatProjectHasTag(project, "bar");
 221     assertThat(result.getTotal()).isEqualTo(1L);
 222     assertThat(result.getSuccess()).isEqualTo(1L);
 223   }
 224
 225   @Test
 226   public void delete_doc_from_index_when_project_is_deleted() {
 227     ComponentDto project = db.components().insertPrivateProject();
 228     indexProject(project, PROJECT_CREATION);
 229     assertThatIndexContainsOnly(project);
 230
 231     db.getDbClient().purgeDao().deleteProject(db.getSession(), project.uuid());
 232     IndexingResult result = indexProject(project, PROJECT_DELETION);
 233
 234     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
 235     assertThat(result.getTotal()).isEqualTo(1L);
 236     assertThat(result.getSuccess()).isEqualTo(1L);
 237   }
 238
 239   @Test
 240   public void do_nothing_if_no_projects_and_apps_to_index() {
 241     // this project should not be indexed
 242     db.components().insertPrivateProject();
 243     db.components().insertPrivateApplication();
 244
 245     underTest.index(db.getSession(), emptyList());
 246
 247     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
 248   }
 249
 250   @Test
 251   public void errors_during_indexing_are_recovered() {
 252     ComponentDto project = db.components().insertPrivateProject();
 253     es.lockWrites(TYPE_PROJECT_MEASURES);
 254
 255     IndexingResult result = indexProject(project, PROJECT_CREATION);
 256     assertThat(result.getTotal()).isEqualTo(1L);
 257     assertThat(result.getFailures()).isEqualTo(1L);
 258
 259     // index is still read-only, fail to recover
 260     result = recover();
 261     assertThat(result.getTotal()).isEqualTo(1L);
 262     assertThat(result.getFailures()).isEqualTo(1L);
 263     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
 264     assertThatEsQueueTableHasSize(1);
 265
 266     es.unlockWrites(TYPE_PROJECT_MEASURES);
 267
 268     result = recover();
 269     assertThat(result.getTotal()).isEqualTo(1L);
 270     assertThat(result.getFailures()).isEqualTo(0L);
 271     assertThatEsQueueTableHasSize(0);
 272     assertThatIndexContainsOnly(project);
 273   }
 274
 275   @Test
 276   public void non_main_branches_are_not_indexed_during_analysis() {
 277     ComponentDto project = db.components().insertPrivateProject();
 278     ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setKey("feature/foo"));
 279
 280     underTest.indexOnAnalysis(branch.uuid());
 281
 282     assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
 283   }
 284
 285   private IndexingResult indexProject(ComponentDto project, ProjectIndexer.Cause cause) {
 286     DbSession dbSession = db.getSession();
 287     Collection<EsQueueDto> items = underTest.prepareForRecovery(dbSession, singletonList(project.uuid()), cause);
 288     dbSession.commit();
 289     return underTest.index(dbSession, items);
 290   }
 291
 292   private void assertThatProjectHasTag(ComponentDto project, String expectedTag) {
 293     SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
 294       .source(new SearchSourceBuilder()
 295         .query(boolQuery()
 296           .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()))
 297           .filter(termQuery(FIELD_TAGS, expectedTag))));
 298
 299     assertThat(es.client().search(request).getHits().getHits())
 300       .extracting(SearchHit::getId)
 301       .contains(project.uuid());
 302   }
 303
 304   private void assertThatEsQueueTableHasSize(int expectedSize) {
 305     assertThat(db.countRowsOfTable("es_queue")).isEqualTo(expectedSize);
 306   }
 307
 308   private void assertThatIndexContainsOnly(SnapshotDto... expectedProjects) {
 309     assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
 310       Arrays.stream(expectedProjects).map(SnapshotDto::getComponentUuid).toArray(String[]::new));
 311   }
 312
 313   private void assertThatIndexContainsOnly(ComponentDto... expectedProjects) {
 314     assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
 315       Arrays.stream(expectedProjects).map(ComponentDto::uuid).toArray(String[]::new));
 316   }
 317
 318   private void assertThatQualifierIs(String qualifier, ComponentDto... expectedComponents) {
 319     String[] expectedComponentUuids = Arrays.stream(expectedComponents).map(ComponentDto::uuid).toArray(String[]::new);
 320     assertThatQualifierIs(qualifier, expectedComponentUuids);
 321   }
 322
 323   private void assertThatQualifierIs(String qualifier, SnapshotDto... expectedComponents) {
 324     String[] expectedComponentUuids = Arrays.stream(expectedComponents).map(SnapshotDto::getComponentUuid).toArray(String[]::new);
 325     assertThatQualifierIs(qualifier, expectedComponentUuids);
 326   }
 327
 328   private void assertThatQualifierIs(String qualifier, String... componentsUuid) {
 329     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
 330       .query(boolQuery()
 331         .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()))
 332         .filter(termQuery(FIELD_QUALIFIER, qualifier))
 333         .filter(termsQuery(FIELD_UUID, componentsUuid)));
 334
 335     SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
 336       .source(searchSourceBuilder);
 337     assertThat(es.client().search(request).getHits().getHits())
 338       .extracting(SearchHit::getId)
 339       .containsExactlyInAnyOrder(componentsUuid);
 340   }
 341
 342   private IndexingResult recover() {
 343     Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), System.currentTimeMillis() + 1_000L, 10);
 344     return underTest.index(db.getSession(), items);
 345   }
 346
 347   private static <T> Consumer<T> defaults() {
 348     return t -> {
 349       // do nothing
 350     };
 351   }
 352
 353 }