From: Teryk Bellahsene Date: Wed, 11 Oct 2017 15:59:13 +0000 (+0200) Subject: SONAR-9928 WS api/components/search relies on Component ES index X-Git-Tag: 6.7-RC1~283 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=ca838c0576e1a785fd4c7ed4609da51c7ba24a62;p=sonarqube.git SONAR-9928 WS api/components/search relies on Component ES index --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java index 206447f5563..fcb2afed0e1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java @@ -22,6 +22,7 @@ package org.sonar.server.component.index; import java.util.Collection; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableCollection; @@ -72,12 +73,12 @@ public class ComponentQuery { // enforce static factory method } - public Builder setOrganization(String organizationUuid) { + public Builder setOrganization(@Nullable String organizationUuid) { this.organizationUuid = organizationUuid; return this; } - public Builder setQuery(String query) { + public Builder setQuery(@Nullable String query) { this.query = query; return this; } @@ -87,7 +88,7 @@ public class ComponentQuery { return this; } - public Builder setLanguage(String language) { + public Builder setLanguage(@Nullable String language) { this.language = language; return this; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchAction.java b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchAction.java index c8c8ef5f1fb..ce509b07410 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchAction.java @@ -31,14 +31,15 @@ import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; import org.sonar.api.server.ws.WebService.Param; import org.sonar.api.utils.Paging; -import org.sonar.api.web.UserRole; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentQuery; import org.sonar.db.organization.OrganizationDto; +import org.sonar.server.component.index.ComponentIndex; +import org.sonar.server.component.index.ComponentQuery; +import org.sonar.server.es.SearchIdResult; +import org.sonar.server.es.SearchOptions; import org.sonar.server.organization.DefaultOrganizationProvider; -import org.sonar.server.user.UserSession; import org.sonar.server.ws.WsUtils; import org.sonarqube.ws.WsComponents; import org.sonarqube.ws.WsComponents.SearchWsResponse; @@ -50,8 +51,8 @@ import static org.sonar.core.util.Protobuf.setNullable; import static org.sonar.core.util.stream.MoreCollectors.toHashSet; import static org.sonar.server.util.LanguageParamUtils.getExampleValue; import static org.sonar.server.util.LanguageParamUtils.getLanguageKeys; -import static org.sonar.server.ws.WsParameterBuilder.createQualifiersParameter; import static org.sonar.server.ws.WsParameterBuilder.QualifierParameterContext.newQualifierParameterContext; +import static org.sonar.server.ws.WsParameterBuilder.createQualifiersParameter; import static org.sonar.server.ws.WsUtils.writeProtobuf; import static org.sonarqube.ws.client.component.ComponentsWsParameters.ACTION_SEARCH; import static org.sonarqube.ws.client.component.ComponentsWsParameters.PARAM_LANGUAGE; @@ -59,19 +60,19 @@ import static org.sonarqube.ws.client.component.ComponentsWsParameters.PARAM_ORG import static org.sonarqube.ws.client.component.ComponentsWsParameters.PARAM_QUALIFIERS; public class SearchAction implements ComponentsWsAction { + private final ComponentIndex componentIndex; private final DbClient dbClient; private final ResourceTypes resourceTypes; private final I18n i18n; - private final UserSession userSession; private final Languages languages; private final DefaultOrganizationProvider defaultOrganizationProvider; - public SearchAction(DbClient dbClient, ResourceTypes resourceTypes, I18n i18n, UserSession userSession, - Languages languages, DefaultOrganizationProvider defaultOrganizationProvider) { + public SearchAction(ComponentIndex componentIndex, DbClient dbClient, ResourceTypes resourceTypes, I18n i18n, Languages languages, + DefaultOrganizationProvider defaultOrganizationProvider) { + this.componentIndex = componentIndex; this.dbClient = dbClient; this.resourceTypes = resourceTypes; this.i18n = i18n; - this.userSession = userSession; this.languages = languages; this.defaultOrganizationProvider = defaultOrganizationProvider; } @@ -127,13 +128,15 @@ public class SearchAction implements ComponentsWsAction { private SearchWsResponse doHandle(SearchWsRequest request) { try (DbSession dbSession = dbClient.openSession(false)) { - ComponentQuery query = buildQuery(request); OrganizationDto organization = getOrganization(dbSession, request); - Paging paging = buildPaging(dbSession, request, organization, query); - List components = searchComponents(dbSession, organization, query, paging); + ComponentQuery esQuery = buildEsQuery(organization, request); + SearchIdResult results = componentIndex.search(esQuery, new SearchOptions().setPage(request.getPage(), request.getPageSize())); + + List components = dbClient.componentDao().selectByUuids(dbSession, results.getIds()); Map projectKeysByUuids = searchProjectsKeysByUuids(dbSession, components); - return buildResponse(components, organization, projectKeysByUuids, paging); + return buildResponse(components, organization, projectKeysByUuids, + Paging.forPageIndex(request.getPage()).withPageSize(request.getPageSize()).andTotal((int) results.getTotal())); } } @@ -145,15 +148,6 @@ public class SearchAction implements ComponentsWsAction { return projects.stream().collect(toMap(ComponentDto::uuid, ComponentDto::getDbKey)); } - private static ComponentQuery buildQuery(SearchWsRequest request) { - List qualifiers = request.getQualifiers(); - return ComponentQuery.builder() - .setNameOrKeyQuery(request.getQuery()) - .setLanguage(request.getLanguage()) - .setQualifiers(qualifiers.toArray(new String[qualifiers.size()])) - .build(); - } - private OrganizationDto getOrganization(DbSession dbSession, SearchWsRequest request) { String organizationKey = Optional.ofNullable(request.getOrganization()) .orElseGet(defaultOrganizationProvider.get()::getKey); @@ -162,16 +156,13 @@ public class SearchAction implements ComponentsWsAction { "No organizationDto with key '%s'", organizationKey); } - private Paging buildPaging(DbSession dbSession, SearchWsRequest request, OrganizationDto organization, ComponentQuery query) { - int total = dbClient.componentDao().countByQuery(dbSession, organization.getUuid(), query); - return Paging.forPageIndex(request.getPage()) - .withPageSize(request.getPageSize()) - .andTotal(total); - } - - private List searchComponents(DbSession dbSession, OrganizationDto organization, ComponentQuery query, Paging paging) { - List componentDtos = dbClient.componentDao().selectByQuery(dbSession, organization.getUuid(), query, paging.offset(), paging.pageSize()); - return userSession.keepAuthorizedComponents(UserRole.USER, componentDtos); + private static ComponentQuery buildEsQuery(OrganizationDto organization, SearchWsRequest request) { + return ComponentQuery.builder() + .setQuery(request.getQuery()) + .setOrganization(organization.getUuid()) + .setLanguage(request.getLanguage()) + .setQualifiers(request.getQualifiers()) + .build(); } private static SearchWsResponse buildResponse(List components, OrganizationDto organization, Map projectKeysByUuids, Paging paging) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java index 99d7d9583f7..b26cecc240c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java @@ -163,5 +163,4 @@ public class ComponentIndexSearchTest { indexer.indexOnStartup(emptySet()); Arrays.stream(components).forEach(c -> authorizationIndexerTester.allowOnlyAnyone(c)); } - } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchActionTest.java index c52742761ac..79689b048f5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchActionTest.java @@ -28,19 +28,25 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.sonar.api.config.internal.MapSettings; import org.sonar.api.resources.Language; import org.sonar.api.resources.Languages; import org.sonar.api.server.ws.WebService; import org.sonar.api.utils.System2; -import org.sonar.api.web.UserRole; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentTesting; import org.sonar.db.component.ResourceTypesRule; import org.sonar.db.organization.OrganizationDto; import org.sonar.db.user.UserDto; +import org.sonar.server.component.index.ComponentIndex; +import org.sonar.server.component.index.ComponentIndexDefinition; +import org.sonar.server.component.index.ComponentIndexer; +import org.sonar.server.es.EsTester; import org.sonar.server.i18n.I18nRule; import org.sonar.server.organization.TestDefaultOrganizationProvider; +import org.sonar.server.permission.index.AuthorizationTypeSupport; +import org.sonar.server.permission.index.PermissionIndexerTester; import org.sonar.server.tester.UserSessionRule; import org.sonar.server.ws.TestRequest; import org.sonar.server.ws.WsActionTester; @@ -50,6 +56,7 @@ import org.sonarqube.ws.WsComponents.SearchWsResponse; import org.sonarqube.ws.client.component.SearchWsRequest; import static java.util.Arrays.asList; +import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -80,12 +87,17 @@ public class SearchActionTest { public UserSessionRule userSession = UserSessionRule.standalone(); @Rule public DbTester db = DbTester.create(System2.INSTANCE); + @Rule + public EsTester es = new EsTester(new ComponentIndexDefinition(new MapSettings().asConfig())); private I18nRule i18n = new I18nRule(); - private TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); private ResourceTypesRule resourceTypes = new ResourceTypesRule(); private Languages languages = mock(Languages.class); + private ComponentIndexer indexer = new ComponentIndexer(db.getDbClient(), es.client()); + private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es, indexer); + private ComponentIndex index = new ComponentIndex(es.client(), new AuthorizationTypeSupport(userSession)); + private UserDto user; private WsActionTester ws; @@ -94,7 +106,7 @@ public class SearchActionTest { public void setUp() { resourceTypes.setAllQualifiers(PROJECT, MODULE, DIRECTORY, FILE); when(languages.all()).thenReturn(javaLanguage()); - ws = new WsActionTester(new SearchAction(db.getDbClient(), resourceTypes, i18n, userSession, languages, defaultOrganizationProvider)); + ws = new WsActionTester(new SearchAction(index, db.getDbClient(), resourceTypes, i18n, languages, defaultOrganizationProvider)); user = db.users().insertUser("john"); userSession.logIn(user); @@ -139,7 +151,7 @@ public class SearchActionTest { ComponentDto file1 = newFileDto(project).setDbKey("file1"); ComponentDto file2 = newFileDto(project).setDbKey("file2"); db.components().insertComponents(project, file1, file2); - setBrowsePermissionOnUser(project); + setBrowsePermissionOnUserAndIndex(project); SearchWsResponse response = call(new SearchWsRequest().setQuery(file1.getDbKey()).setQualifiers(singletonList(FILE))); @@ -180,11 +192,13 @@ public class SearchActionTest { ComponentDto project2 = ComponentTesting.newPrivateProjectDto(db.getDefaultOrganization()); ComponentDto file3 = newFileDto(project2).setDbKey("file3"); db.components().insertComponents(project1, file1, file2, project2, file3); - setBrowsePermissionOnUser(project1); + setBrowsePermissionOnUserAndIndex(project1); SearchWsResponse response = call(new SearchWsRequest().setQualifiers(singletonList(FILE))); - assertThat(response.getComponentsList()).extracting(Component::getKey).containsOnly(file1.getDbKey(), file2.getDbKey()); + assertThat(response.getComponentsList()).extracting(Component::getKey) + .containsExactlyInAnyOrder(file1.getDbKey(), file2.getDbKey()); + assertThat(response.getPaging().getTotal()).isEqualTo(2); } @Test @@ -195,6 +209,7 @@ public class SearchActionTest { ComponentDto file2 = newFileDto(module).setDbKey("file2"); ComponentDto file3 = newFileDto(project).setDbKey("file3"); db.components().insertComponents(project, module, file1, file2, file3); + setBrowsePermissionOnUserAndIndex(project); SearchWsResponse response = call(new SearchWsRequest().setQualifiers(asList(PROJECT, MODULE, FILE))); @@ -206,29 +221,11 @@ public class SearchActionTest { tuple(file3.getDbKey(), project.getDbKey())); } - @Test - public void do_not_verify_permissions_if_user_is_root() throws IOException { - OrganizationDto org = db.organizations().insert(); - ComponentDto project1 = ComponentTesting.newPrivateProjectDto(org); - ComponentDto file1 = newFileDto(project1); - ComponentDto project2 = ComponentTesting.newPrivateProjectDto(org); - ComponentDto file2 = newFileDto(project2); - db.components().insertComponents(project1, file1, project2, file2); - - SearchWsRequest request = new SearchWsRequest().setQualifiers(singletonList(FILE)).setOrganization(org.getKey()); - - userSession.logIn().setNonRoot(); - assertThat(call(request).getComponentsCount()).isZero(); - - userSession.logIn().setRoot(); - assertThat(call(request).getComponentsList()).extracting(Component::getKey).containsOnly(file1.getDbKey(), file2.getDbKey()); - } - @Test public void does_not_return_branches() { ComponentDto project = db.components().insertMainBranch(); ComponentDto branch = db.components().insertProjectBranch(project); - userSession.logIn().setRoot(); + setBrowsePermissionOnUserAndIndex(project, branch); SearchWsResponse response = call(new SearchWsRequest().setQualifiers(asList(PROJECT, MODULE, FILE))); @@ -261,7 +258,7 @@ public class SearchActionTest { ComponentDto directory = newDirectory(module, "path/to/directoy").setUuid("directory-uuid").setDbKey("directory-key").setName("Directory Name"); db.components().insertComponents(project, module, directory, newFileDto(module, directory, "file-uuid").setDbKey("file-key").setLanguage("java").setName("File Name")); - userSession.addProjectPermission(UserRole.USER, project); + setBrowsePermissionOnUserAndIndex(project); String response = ws.newRequest() .setMediaType(MediaTypes.JSON) @@ -273,12 +270,13 @@ public class SearchActionTest { private void insertProjectsAuthorizedForUser(ComponentDto... projects) { db.components().insertComponents(projects); - setBrowsePermissionOnUser(projects); + setBrowsePermissionOnUserAndIndex(projects); db.commit(); } - private void setBrowsePermissionOnUser(ComponentDto... projects) { - Arrays.stream(projects).forEach(project -> userSession.addProjectPermission(UserRole.USER, project)); + private void setBrowsePermissionOnUserAndIndex(ComponentDto... projects) { + index(); + Arrays.stream(projects).forEach(project -> authorizationIndexerTester.allowOnlyUser(project, user)); } private SearchWsResponse call(SearchWsRequest wsRequest) { @@ -292,6 +290,10 @@ public class SearchActionTest { return request.executeProtobuf(SearchWsResponse.class); } + private void index() { + indexer.indexOnStartup(emptySet()); + } + private static Language[] javaLanguage() { return new Language[] {new Language() { @Override