diff options
author | Aurelien Poscia <aurelien.poscia@sonarsource.com> | 2023-07-05 11:27:38 +0200 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2023-07-18 20:03:21 +0000 |
commit | 781f6976f9b44bec7d2a545f2407cabdfbf4c7a2 (patch) | |
tree | e7c6cdec02e4a7bcf0941586937997e3c1dfd6ff /server/sonar-webserver-es | |
parent | a82999c5d8b9ab38e20950d5ecd2c2859e29d2ad (diff) | |
download | sonarqube-781f6976f9b44bec7d2a545f2407cabdfbf4c7a2.tar.gz sonarqube-781f6976f9b44bec7d2a545f2407cabdfbf4c7a2.zip |
SONAR-19784 allow the CE to re-compute permissions
Diffstat (limited to 'server/sonar-webserver-es')
-rw-r--r-- | server/sonar-webserver-es/src/it/java/org/sonar/server/permission/index/PermissionIndexerDaoIT.java | 272 | ||||
-rw-r--r-- | server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java | 195 | ||||
-rw-r--r-- | server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java | 180 | ||||
-rw-r--r-- | server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java | 424 | ||||
-rw-r--r-- | server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndex.java (renamed from server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java) | 0 | ||||
-rw-r--r-- | server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndexer.java (renamed from server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java) | 0 |
6 files changed, 0 insertions, 1071 deletions
diff --git a/server/sonar-webserver-es/src/it/java/org/sonar/server/permission/index/PermissionIndexerDaoIT.java b/server/sonar-webserver-es/src/it/java/org/sonar/server/permission/index/PermissionIndexerDaoIT.java deleted file mode 100644 index 8bc2b38bc0f..00000000000 --- a/server/sonar-webserver-es/src/it/java/org/sonar/server/permission/index/PermissionIndexerDaoIT.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.assertj.core.api.Assertions; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.System2; -import org.sonar.core.util.Uuids; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.DbTester; -import org.sonar.db.component.ProjectData; -import org.sonar.db.permission.GroupPermissionDto; -import org.sonar.db.portfolio.PortfolioDto; -import org.sonar.db.project.ProjectDto; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.UserDbTester; -import org.sonar.db.user.UserDto; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.api.resources.Qualifiers.APP; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.resources.Qualifiers.VIEW; -import static org.sonar.api.web.UserRole.ADMIN; -import static org.sonar.api.web.UserRole.USER; - -public class PermissionIndexerDaoIT { - - @Rule - public DbTester dbTester = DbTester.create(System2.INSTANCE); - - private final DbClient dbClient = dbTester.getDbClient(); - private final DbSession dbSession = dbTester.getSession(); - private final UserDbTester userDbTester = new UserDbTester(dbTester); - - private ProjectDto publicProject; - private ProjectDto privateProject1; - private ProjectDto privateProject2; - private PortfolioDto view1; - private PortfolioDto view2; - private ProjectDto application; - private UserDto user1; - private UserDto user2; - private GroupDto group; - - private final PermissionIndexerDao underTest = new PermissionIndexerDao(); - - @Before - public void setUp() { - publicProject = dbTester.components().insertPublicProject().getProjectDto(); - privateProject1 = dbTester.components().insertPrivateProject().getProjectDto(); - privateProject2 = dbTester.components().insertPrivateProject().getProjectDto(); - view1 = dbTester.components().insertPublicPortfolioDto(); - view2 = dbTester.components().insertPublicPortfolioDto(); - application = dbTester.components().insertPublicApplication().getProjectDto(); - user1 = userDbTester.insertUser(); - user2 = userDbTester.insertUser(); - group = userDbTester.insertGroup(); - } - - @Test - public void select_all() { - insertTestDataForProjectsAndViews(); - - Collection<IndexPermissions> dtos = underTest.selectAll(dbClient, dbSession); - Assertions.assertThat(dtos).hasSize(6); - - IndexPermissions publicProjectAuthorization = getByProjectUuid(publicProject.getUuid(), dtos); - isPublic(publicProjectAuthorization, PROJECT); - - IndexPermissions view1Authorization = getByProjectUuid(view1.getUuid(), dtos); - isPublic(view1Authorization, VIEW); - - IndexPermissions applicationAuthorization = getByProjectUuid(application.getUuid(), dtos); - isPublic(applicationAuthorization, APP); - - IndexPermissions privateProject1Authorization = getByProjectUuid(privateProject1.getUuid(), dtos); - assertThat(privateProject1Authorization.getGroupUuids()).containsOnly(group.getUuid()); - assertThat(privateProject1Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject1Authorization.getUserUuids()).containsOnly(user1.getUuid(), user2.getUuid()); - assertThat(privateProject1Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions privateProject2Authorization = getByProjectUuid(privateProject2.getUuid(), dtos); - assertThat(privateProject2Authorization.getGroupUuids()).isEmpty(); - assertThat(privateProject2Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject2Authorization.getUserUuids()).containsOnly(user1.getUuid()); - assertThat(privateProject2Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions view2Authorization = getByProjectUuid(view2.getUuid(), dtos); - isPublic(view2Authorization, VIEW); - } - - @Test - public void selectByUuids() { - insertTestDataForProjectsAndViews(); - - Map<String, IndexPermissions> dtos = underTest - .selectByUuids(dbClient, dbSession, - asList(publicProject.getUuid(), privateProject1.getUuid(), privateProject2.getUuid(), view1.getUuid(), view2.getUuid(), application.getUuid())) - .stream() - .collect(Collectors.toMap(IndexPermissions::getEntityUuid, Function.identity())); - Assertions.assertThat(dtos).hasSize(6); - - IndexPermissions publicProjectAuthorization = dtos.get(publicProject.getUuid()); - isPublic(publicProjectAuthorization, PROJECT); - - IndexPermissions view1Authorization = dtos.get(view1.getUuid()); - isPublic(view1Authorization, VIEW); - - IndexPermissions applicationAuthorization = dtos.get(application.getUuid()); - isPublic(applicationAuthorization, APP); - - IndexPermissions privateProject1Authorization = dtos.get(privateProject1.getUuid()); - assertThat(privateProject1Authorization.getGroupUuids()).containsOnly(group.getUuid()); - assertThat(privateProject1Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject1Authorization.getUserUuids()).containsOnly(user1.getUuid(), user2.getUuid()); - assertThat(privateProject1Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions privateProject2Authorization = dtos.get(privateProject2.getUuid()); - assertThat(privateProject2Authorization.getGroupUuids()).isEmpty(); - assertThat(privateProject2Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject2Authorization.getUserUuids()).containsOnly(user1.getUuid()); - assertThat(privateProject2Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions view2Authorization = dtos.get(view2.getUuid()); - isPublic(view2Authorization, VIEW); - } - - @Test - public void selectByUuids_returns_empty_list_when_project_does_not_exist() { - insertTestDataForProjectsAndViews(); - - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList("missing")); - Assertions.assertThat(dtos).isEmpty(); - } - - @Test - public void select_by_projects_with_high_number_of_projects() { - List<String> projectUuids = new ArrayList<>(); - for (int i = 0; i < 3500; i++) { - ProjectData project = dbTester.components().insertPrivateProject(Integer.toString(i)); - projectUuids.add(project.projectUuid()); - GroupPermissionDto dto = new GroupPermissionDto() - .setUuid(Uuids.createFast()) - .setGroupUuid(group.getUuid()) - .setGroupName(group.getName()) - .setRole(USER) - .setEntityUuid(project.projectUuid()) - .setEntityName(project.getProjectDto().getName()); - dbClient.groupPermissionDao().insert(dbSession, dto, project.getProjectDto(), null); - } - dbSession.commit(); - - assertThat(underTest.selectByUuids(dbClient, dbSession, projectUuids)) - .hasSize(3500) - .extracting(IndexPermissions::getEntityUuid) - .containsAll(projectUuids); - } - - @Test - public void return_private_project_without_any_permission_when_no_permission_in_DB() { - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(privateProject1.getUuid())); - - // no permissions - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupUuids()).isEmpty(); - assertThat(dto.getUserUuids()).isEmpty(); - assertThat(dto.isAllowAnyone()).isFalse(); - assertThat(dto.getEntityUuid()).isEqualTo(privateProject1.getUuid()); - assertThat(dto.getQualifier()).isEqualTo(privateProject1.getQualifier()); - } - - @Test - public void return_public_project_with_only_AllowAnyone_true_when_no_permission_in_DB() { - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(publicProject.getUuid())); - - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupUuids()).isEmpty(); - assertThat(dto.getUserUuids()).isEmpty(); - assertThat(dto.isAllowAnyone()).isTrue(); - assertThat(dto.getEntityUuid()).isEqualTo(publicProject.getUuid()); - assertThat(dto.getQualifier()).isEqualTo(publicProject.getQualifier()); - } - - @Test - public void return_private_project_with_AllowAnyone_false_and_user_id_when_user_is_granted_USER_permission_directly() { - dbTester.users().insertProjectPermissionOnUser(user1, USER, privateProject1); - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(privateProject1.getUuid())); - - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupUuids()).isEmpty(); - assertThat(dto.getUserUuids()).containsOnly(user1.getUuid()); - assertThat(dto.isAllowAnyone()).isFalse(); - assertThat(dto.getEntityUuid()).isEqualTo(privateProject1.getUuid()); - assertThat(dto.getQualifier()).isEqualTo(privateProject1.getQualifier()); - } - - @Test - public void return_private_project_with_AllowAnyone_false_and_group_id_but_not_user_id_when_user_is_granted_USER_permission_through_group() { - dbTester.users().insertMember(group, user1); - dbTester.users().insertEntityPermissionOnGroup(group, USER, privateProject1); - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(privateProject1.getUuid())); - - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupUuids()).containsOnly(group.getUuid()); - assertThat(dto.getUserUuids()).isEmpty(); - assertThat(dto.isAllowAnyone()).isFalse(); - assertThat(dto.getEntityUuid()).isEqualTo(privateProject1.getUuid()); - assertThat(dto.getQualifier()).isEqualTo(privateProject1.getQualifier()); - } - - private void isPublic(IndexPermissions view1Authorization, String qualifier) { - assertThat(view1Authorization.getGroupUuids()).isEmpty(); - assertThat(view1Authorization.isAllowAnyone()).isTrue(); - assertThat(view1Authorization.getUserUuids()).isEmpty(); - assertThat(view1Authorization.getQualifier()).isEqualTo(qualifier); - } - - private static IndexPermissions getByProjectUuid(String projectUuid, Collection<IndexPermissions> dtos) { - return dtos.stream().filter(dto -> dto.getEntityUuid().equals(projectUuid)).findFirst().orElseThrow(IllegalArgumentException::new); - } - - private void insertTestDataForProjectsAndViews() { - // user1 has USER access on both private projects - userDbTester.insertProjectPermissionOnUser(user1, ADMIN, publicProject); - userDbTester.insertProjectPermissionOnUser(user1, USER, privateProject1); - userDbTester.insertProjectPermissionOnUser(user1, USER, privateProject2); - userDbTester.insertProjectPermissionOnUser(user1, ADMIN, view1); - userDbTester.insertProjectPermissionOnUser(user1, ADMIN, application); - - // user2 has USER access on privateProject1 only - userDbTester.insertProjectPermissionOnUser(user2, USER, privateProject1); - userDbTester.insertProjectPermissionOnUser(user2, ADMIN, privateProject2); - - // group1 has USER access on privateProject1 only - userDbTester.insertEntityPermissionOnGroup(group, USER, privateProject1); - userDbTester.insertEntityPermissionOnGroup(group, ADMIN, privateProject1); - userDbTester.insertEntityPermissionOnGroup(group, ADMIN, view1); - userDbTester.insertEntityPermissionOnGroup(group, ADMIN, application); - } -} diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java deleted file mode 100644 index 55100bfbb4d..00000000000 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableSet; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.es.EsQueueDto; -import org.sonar.server.es.BulkIndexer; -import org.sonar.server.es.BulkIndexer.Size; -import org.sonar.server.es.EsClient; -import org.sonar.server.es.EventIndexer; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.Indexers; -import org.sonar.server.es.IndexingResult; -import org.sonar.server.es.OneToOneResilientIndexingListener; -import org.springframework.beans.factory.annotation.Autowired; - -import static java.util.Collections.emptyList; - -/** - * Populates the types "authorization" of each index requiring entity - * authorization. - */ -public class PermissionIndexer implements EventIndexer { - private final DbClient dbClient; - private final EsClient esClient; - private final Collection<AuthorizationScope> authorizationScopes; - private final Map<String, IndexType> indexTypeByFormat; - - @Autowired(required = false) - public PermissionIndexer(DbClient dbClient, EsClient esClient, NeedAuthorizationIndexer... needAuthorizationIndexers) { - this(dbClient, esClient, Arrays.stream(needAuthorizationIndexers) - .map(NeedAuthorizationIndexer::getAuthorizationScope) - .toList()); - } - - @VisibleForTesting - @Autowired(required = false) - public PermissionIndexer(DbClient dbClient, EsClient esClient, Collection<AuthorizationScope> authorizationScopes) { - this.dbClient = dbClient; - this.esClient = esClient; - this.authorizationScopes = authorizationScopes; - this.indexTypeByFormat = authorizationScopes.stream() - .map(AuthorizationScope::getIndexType) - .collect(Collectors.toMap(IndexType.IndexMainType::format, Function.identity())); - } - - @Override - public Set<IndexType> getIndexTypes() { - return ImmutableSet.copyOf(indexTypeByFormat.values()); - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - // TODO do not load everything in memory. Db rows should be scrolled. - List<IndexPermissions> authorizations = getAllAuthorizations(); - Stream<AuthorizationScope> scopes = getScopes(uninitializedIndexTypes); - index(authorizations, scopes, Size.LARGE); - } - - public void indexAll(Set<IndexType> uninitializedIndexTypes) { - // TODO do not load everything in memory. Db rows should be scrolled. - List<IndexPermissions> authorizations = getAllAuthorizations(); - Stream<AuthorizationScope> scopes = getScopes(uninitializedIndexTypes); - index(authorizations, scopes, Size.REGULAR); - } - - @VisibleForTesting - void index(List<IndexPermissions> authorizations) { - index(authorizations, authorizationScopes.stream(), Size.REGULAR); - } - - @Override - public Collection<EsQueueDto> prepareForRecoveryOnEntityEvent(DbSession dbSession, Collection<String> entityUuids, Indexers.EntityEvent cause) { - return switch (cause) { - case PROJECT_KEY_UPDATE, PROJECT_TAGS_UPDATE -> - // nothing to change. project key and tags are not part of this index - emptyList(); - case CREATION, DELETION, PERMISSION_CHANGE -> insertIntoEsQueue(dbSession, entityUuids); - }; - } - - @Override - public Collection<EsQueueDto> prepareForRecoveryOnBranchEvent(DbSession dbSession, Collection<String> branchUuids, Indexers.BranchEvent cause) { - return emptyList(); - } - - private Collection<EsQueueDto> insertIntoEsQueue(DbSession dbSession, Collection<String> projectUuids) { - List<EsQueueDto> items = indexTypeByFormat.values().stream() - .flatMap(indexType -> projectUuids.stream().map(projectUuid -> EsQueueDto.create(indexType.format(), AuthorizationDoc.idOf(projectUuid), null, projectUuid))) - .toList(); - - dbClient.esQueueDao().insert(dbSession, items); - return items; - } - - private void index(Collection<IndexPermissions> authorizations, Stream<AuthorizationScope> scopes, Size bulkSize) { - if (authorizations.isEmpty()) { - return; - } - - // index each authorization in each scope - scopes.forEach(scope -> { - IndexType indexType = scope.getIndexType(); - - BulkIndexer bulkIndexer = new BulkIndexer(esClient, indexType, bulkSize); - bulkIndexer.start(); - - authorizations.stream() - .filter(scope.getEntityPredicate()) - .map(dto -> AuthorizationDoc.fromDto(indexType, dto).toIndexRequest()) - .forEach(bulkIndexer::add); - - bulkIndexer.stop(); - }); - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - IndexingResult result = new IndexingResult(); - - List<BulkIndexer> bulkIndexers = items.stream() - .map(EsQueueDto::getDocType) - .distinct() - .map(indexTypeByFormat::get) - .filter(Objects::nonNull) - .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) - .toList(); - - if (bulkIndexers.isEmpty()) { - return result; - } - - bulkIndexers.forEach(BulkIndexer::start); - - PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); - Set<String> remainingEntityUuids = items.stream().map(EsQueueDto::getDocId) - .map(AuthorizationDoc::entityUuidOf) - .collect(Collectors.toSet()); - permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingEntityUuids).forEach(p -> { - remainingEntityUuids.remove(p.getEntityUuid()); - bulkIndexers.forEach(bi -> bi.add(AuthorizationDoc.fromDto(bi.getIndexType(), p).toIndexRequest())); - }); - - // the remaining references on entities that don't exist in db. They must - // be deleted from the index. - remainingEntityUuids.forEach(entityUuid -> bulkIndexers.forEach(bi -> { - String authorizationDocId = AuthorizationDoc.idOf(entityUuid); - bi.addDeletion(bi.getIndexType(), authorizationDocId, authorizationDocId); - })); - - bulkIndexers.forEach(b -> result.add(b.stop())); - - return result; - } - - private Stream<AuthorizationScope> getScopes(Set<IndexType> indexTypes) { - return authorizationScopes.stream() - .filter(scope -> indexTypes.contains(scope.getIndexType())); - } - - private List<IndexPermissions> getAllAuthorizations() { - try (DbSession dbSession = dbClient.openSession(false)) { - return new PermissionIndexerDao().selectAll(dbClient, dbSession); - } - } -} diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java deleted file mode 100644 index 14656ebf5c8..00000000000 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import com.google.common.collect.ImmutableList; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang.StringUtils; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; - -import static org.apache.commons.lang.StringUtils.repeat; -import static org.sonar.db.DatabaseUtils.executeLargeInputs; - -/** - * No streaming because of union of joins -> no need to use ResultSetIterator - */ -public class PermissionIndexerDao { - - private enum RowKind { - USER, GROUP, ANYONE, NONE - } - - private static final String SQL_TEMPLATE = """ - with entity as ((select prj.uuid as uuid, - prj.private as isPrivate, - prj.qualifier as qualifier - from projects prj) - union - (select p.uuid as uuid, - p.private as isPrivate, - 'VW' as qualifier - from portfolios p - where p.parent_uuid is null)) - SELECT entity_authorization.kind as kind, - entity_authorization.entity as entity, - entity_authorization.user_uuid as user_uuid, - entity_authorization.group_uuid as group_uuid, - entity_authorization.qualifier as qualifier - FROM (SELECT '%s' as kind, - e.uuid AS entity, - e.qualifier AS qualifier, - user_roles.user_uuid AS user_uuid, - NULL AS group_uuid - FROM entity e - INNER JOIN user_roles ON user_roles.entity_uuid = e.uuid AND user_roles.role = 'user' - WHERE (1 = 1) - {entitiesCondition} - UNION - SELECT '%s' as kind, e.uuid AS entity, e.qualifier AS qualifier, NULL AS user_uuid, groups.uuid AS group_uuid - FROM entity e - INNER JOIN group_roles - ON group_roles.entity_uuid = e.uuid AND group_roles.role = 'user' - INNER JOIN groups ON groups.uuid = group_roles.group_uuid - WHERE group_uuid IS NOT NULL - {entitiesCondition} - UNION - SELECT '%s' as kind, e.uuid AS entity, e.qualifier AS qualifier, NULL AS user_uuid, NULL AS group_uuid - FROM entity e - WHERE e.isPrivate = ? - {entitiesCondition} - UNION - SELECT '%s' as kind, e.uuid AS entity, e.qualifier AS qualifier, NULL AS user_uuid, NULL AS group_uuid - FROM entity e - WHERE e.isPrivate = ? - {entitiesCondition} - ) entity_authorization""".formatted(RowKind.USER, RowKind.GROUP, RowKind.ANYONE, RowKind.NONE); - - List<IndexPermissions> selectAll(DbClient dbClient, DbSession session) { - return doSelectByEntities(dbClient, session, Collections.emptyList()); - } - - public List<IndexPermissions> selectByUuids(DbClient dbClient, DbSession session, Collection<String> entitiesUuid) { - // we use a smaller partitionSize because the SQL_TEMPLATE contain 4x the list of entity uuid. - // the MsSQL jdbc driver accept a maximum of 2100 prepareStatement parameter. To stay under the limit, - // we go with batch of 1000/2=500 entities uuids, to stay under the limit (4x500 < 2100) - return executeLargeInputs(entitiesUuid, entity -> doSelectByEntities(dbClient, session, entity), i -> i / 2); - } - - private static List<IndexPermissions> doSelectByEntities(DbClient dbClient, DbSession session, List<String> entitiesUuids) { - try { - Map<String, IndexPermissions> dtosByEntityUuid = new HashMap<>(); - try (PreparedStatement stmt = createStatement(dbClient, session, entitiesUuids); - ResultSet rs = stmt.executeQuery()) { - while (rs.next()) { - processRow(rs, dtosByEntityUuid); - } - return ImmutableList.copyOf(dtosByEntityUuid.values()); - } - } catch (SQLException e) { - throw new IllegalStateException("Fail to select authorizations", e); - } - } - - private static PreparedStatement createStatement(DbClient dbClient, DbSession session, List<String> entityUuids) throws SQLException { - String sql; - if (entityUuids.isEmpty()) { - sql = StringUtils.replace(SQL_TEMPLATE, "{entitiesCondition}", ""); - } else { - sql = StringUtils.replace(SQL_TEMPLATE, "{entitiesCondition}", " AND e.uuid in (" + repeat("?", ", ", entityUuids.size()) + ")"); - } - PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); - int index = 1; - // query for RowKind.USER - index = populateEntityUuidPlaceholders(stmt, entityUuids, index); - // query for RowKind.GROUP - index = populateEntityUuidPlaceholders(stmt, entityUuids, index); - // query for RowKind.ANYONE - index = setPrivateEntityPlaceHolder(stmt, index, false); - index = populateEntityUuidPlaceholders(stmt, entityUuids, index); - // query for RowKind.NONE - index = setPrivateEntityPlaceHolder(stmt, index, true); - populateEntityUuidPlaceholders(stmt, entityUuids, index); - return stmt; - } - - private static int populateEntityUuidPlaceholders(PreparedStatement stmt, List<String> entityUuids, int index) throws SQLException { - int newIndex = index; - for (String entityUuid : entityUuids) { - stmt.setString(newIndex, entityUuid); - newIndex++; - } - return newIndex; - } - - private static int setPrivateEntityPlaceHolder(PreparedStatement stmt, int index, boolean isPrivate) throws SQLException { - int newIndex = index; - stmt.setBoolean(newIndex, isPrivate); - newIndex++; - return newIndex; - } - - private static void processRow(ResultSet rs, Map<String, IndexPermissions> dtosByEntityUuid) throws SQLException { - RowKind rowKind = RowKind.valueOf(rs.getString(1)); - String entityUuid = rs.getString(2); - - IndexPermissions dto = dtosByEntityUuid.get(entityUuid); - if (dto == null) { - String qualifier = rs.getString(5); - dto = new IndexPermissions(entityUuid, qualifier); - dtosByEntityUuid.put(entityUuid, dto); - } - switch (rowKind) { - case NONE: - break; - case USER: - dto.addUserUuid(rs.getString(3)); - break; - case GROUP: - dto.addGroupUuid(rs.getString(4)); - break; - case ANYONE: - dto.allowAnyone(); - break; - } - } -} diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java deleted file mode 100644 index 7e93ed7f4d8..00000000000 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java +++ /dev/null @@ -1,424 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import java.util.Collection; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.System2; -import org.sonar.db.DbSession; -import org.sonar.db.DbTester; -import org.sonar.db.component.ProjectData; -import org.sonar.db.entity.EntityDto; -import org.sonar.db.es.EsQueueDto; -import org.sonar.db.portfolio.PortfolioDto; -import org.sonar.db.project.ProjectDto; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.UserDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.IndexType.IndexMainType; -import org.sonar.server.es.Indexers.EntityEvent; -import org.sonar.server.es.IndexingResult; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.web.UserRole.ADMIN; -import static org.sonar.api.web.UserRole.USER; -import static org.sonar.server.es.Indexers.EntityEvent.PERMISSION_CHANGE; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; - -public class PermissionIndexerTest { - - private static final IndexMainType INDEX_TYPE_FOO_AUTH = IndexType.main(FooIndexDefinition.DESCRIPTOR, TYPE_AUTHORIZATION); - - @Rule - public DbTester db = DbTester.create(System2.INSTANCE); - @Rule - public EsTester es = EsTester.createCustom(new FooIndexDefinition()); - @Rule - public UserSessionRule userSession = UserSessionRule.standalone(); - - private FooIndex fooIndex = new FooIndex(es.client(), new WebAuthorizationTypeSupport(userSession)); - private FooIndexer fooIndexer = new FooIndexer(es.client(), db.getDbClient()); - private PermissionIndexer underTest = new PermissionIndexer(db.getDbClient(), es.client(), fooIndexer); - - @Test - public void indexOnStartup_grants_access_to_any_user_and_to_group_Anyone_on_public_projects() { - ProjectDto project = createAndIndexPublicProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - - indexOnStartup(); - - verifyAnyoneAuthorized(project); - verifyAuthorized(project, user1); - verifyAuthorized(project, user2); - } - - @Test - public void indexAll_grants_access_to_any_user_and_to_group_Anyone_on_public_projects() { - ProjectDto project = createAndIndexPublicProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - - underTest.indexAll(underTest.getIndexTypes()); - - verifyAnyoneAuthorized(project); - verifyAuthorized(project, user1); - verifyAuthorized(project, user2); - } - - @Test - public void deletion_resilience_will_deindex_projects() { - ProjectDto project1 = createUnindexedPublicProject(); - ProjectDto project2 = createUnindexedPublicProject(); - // UserDto user1 = db.users().insertUser(); - indexOnStartup(); - assertThat(es.countDocuments(INDEX_TYPE_FOO_AUTH)).isEqualTo(2); - - // Simulate a indexation issue - db.getDbClient().purgeDao().deleteProject(db.getSession(), project1.getUuid(), PROJECT, project1.getName(), project1.getKey()); - underTest.prepareForRecoveryOnEntityEvent(db.getSession(), asList(project1.getUuid()), EntityEvent.DELETION); - assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isOne(); - Collection<EsQueueDto> esQueueDtos = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), Long.MAX_VALUE, 2); - - underTest.index(db.getSession(), esQueueDtos); - - assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isZero(); - assertThat(es.countDocuments(INDEX_TYPE_FOO_AUTH)).isOne(); - } - - @Test - public void indexOnStartup_grants_access_to_user() { - ProjectDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user1, USER, project); - db.users().insertProjectPermissionOnUser(user2, ADMIN, project); - - indexOnStartup(); - - // anonymous - verifyAnyoneNotAuthorized(project); - - // user1 has access - verifyAuthorized(project, user1); - - // user2 has not access (only USER permission is accepted) - verifyNotAuthorized(project, user2); - } - - @Test - public void indexOnStartup_grants_access_to_group_on_private_project() { - ProjectDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - UserDto user3 = db.users().insertUser(); - GroupDto group1 = db.users().insertGroup(); - GroupDto group2 = db.users().insertGroup(); - db.users().insertEntityPermissionOnGroup(group1, USER, project); - db.users().insertEntityPermissionOnGroup(group2, ADMIN, project); - - indexOnStartup(); - - // anonymous - verifyAnyoneNotAuthorized(project); - - // group1 has access - verifyAuthorized(project, user1, group1); - - // group2 has not access (only USER permission is accepted) - verifyNotAuthorized(project, user2, group2); - - // user3 is not in any group - verifyNotAuthorized(project, user3); - } - - @Test - public void indexOnStartup_grants_access_to_user_and_group() { - ProjectDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - db.users().insertMember(group, user2); - db.users().insertProjectPermissionOnUser(user1, USER, project); - db.users().insertEntityPermissionOnGroup(group, USER, project); - - indexOnStartup(); - - // anonymous - verifyAnyoneNotAuthorized(project); - - // has direct access - verifyAuthorized(project, user1); - - // has access through group - verifyAuthorized(project, user1, group); - - // no access - verifyNotAuthorized(project, user2); - } - - @Test - public void indexOnStartup_does_not_grant_access_to_anybody_on_private_project() { - ProjectDto project = createAndIndexPrivateProject(); - UserDto user = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - - indexOnStartup(); - - verifyAnyoneNotAuthorized(project); - verifyNotAuthorized(project, user); - verifyNotAuthorized(project, user, group); - } - - @Test - public void indexOnStartup_grants_access_to_anybody_on_public_project() { - ProjectDto project = createAndIndexPublicProject(); - UserDto user = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - - indexOnStartup(); - - verifyAnyoneAuthorized(project); - verifyAuthorized(project, user); - verifyAuthorized(project, user, group); - } - - @Test - public void indexOnStartup_grants_access_to_anybody_on_view() { - PortfolioDto view = createAndIndexPortfolio(); - UserDto user = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - - indexOnStartup(); - - verifyAnyoneAuthorized(view); - verifyAuthorized(view, user); - verifyAuthorized(view, user, group); - } - - @Test - public void indexOnStartup_grants_access_on_many_projects() { - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - ProjectDto project = null; - for (int i = 0; i < 10; i++) { - project = createAndIndexPrivateProject(); - db.users().insertProjectPermissionOnUser(user1, USER, project); - } - - indexOnStartup(); - - verifyAnyoneNotAuthorized(project); - verifyAuthorized(project, user1); - verifyNotAuthorized(project, user2); - } - - @Test - public void public_projects_are_visible_to_anybody() { - ProjectDto projectOnOrg1 = createAndIndexPublicProject(); - UserDto user = db.users().insertUser(); - - indexOnStartup(); - - verifyAnyoneAuthorized(projectOnOrg1); - verifyAuthorized(projectOnOrg1, user); - } - - @Test - public void permissions_are_not_updated_on_project_tags_update() { - ProjectDto project = createAndIndexPublicProject(); - - indexPermissions(project, EntityEvent.PROJECT_TAGS_UPDATE); - - assertThatAuthIndexHasSize(0); - verifyAnyoneNotAuthorized(project); - } - - @Test - public void permissions_are_not_updated_on_project_key_update() { - ProjectDto project = createAndIndexPublicProject(); - - indexPermissions(project, EntityEvent.PROJECT_TAGS_UPDATE); - - assertThatAuthIndexHasSize(0); - verifyAnyoneNotAuthorized(project); - } - - @Test - public void index_permissions_on_project_creation() { - ProjectDto project = createAndIndexPrivateProject(); - UserDto user = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user, USER, project); - - indexPermissions(project, EntityEvent.CREATION); - - assertThatAuthIndexHasSize(1); - verifyAuthorized(project, user); - } - - @Test - public void index_permissions_on_permission_change() { - ProjectDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user1, USER, project); - indexPermissions(project, EntityEvent.CREATION); - verifyAuthorized(project, user1); - verifyNotAuthorized(project, user2); - - db.users().insertProjectPermissionOnUser(user2, USER, project); - indexPermissions(project, PERMISSION_CHANGE); - - verifyAuthorized(project, user1); - verifyAuthorized(project, user1); - } - - @Test - public void delete_permissions_on_project_deletion() { - ProjectDto project = createAndIndexPrivateProject(); - UserDto user = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user, USER, project); - indexPermissions(project, EntityEvent.CREATION); - verifyAuthorized(project, user); - - db.getDbClient().purgeDao().deleteProject(db.getSession(), project.getUuid(), PROJECT, project.getUuid(), project.getKey()); - indexPermissions(project, EntityEvent.DELETION); - - verifyNotAuthorized(project, user); - assertThatAuthIndexHasSize(0); - } - - @Test - public void errors_during_indexing_are_recovered() { - ProjectDto project = createAndIndexPublicProject(); - es.lockWrites(INDEX_TYPE_FOO_AUTH); - - IndexingResult result = indexPermissions(project, PERMISSION_CHANGE); - assertThat(result.getTotal()).isOne(); - assertThat(result.getFailures()).isOne(); - - // index is still read-only, fail to recover - result = recover(); - assertThat(result.getTotal()).isOne(); - assertThat(result.getFailures()).isOne(); - assertThatAuthIndexHasSize(0); - assertThatEsQueueTableHasSize(1); - - es.unlockWrites(INDEX_TYPE_FOO_AUTH); - - result = recover(); - assertThat(result.getTotal()).isOne(); - assertThat(result.getFailures()).isZero(); - verifyAnyoneAuthorized(project); - assertThatEsQueueTableHasSize(0); - } - - private void assertThatAuthIndexHasSize(int expectedSize) { - assertThat(es.countDocuments(FooIndexDefinition.TYPE_AUTHORIZATION)).isEqualTo(expectedSize); - } - - private void indexOnStartup() { - underTest.indexOnStartup(underTest.getIndexTypes()); - } - - private void verifyAuthorized(EntityDto entity, UserDto user) { - logIn(user); - verifyAuthorized(entity, true); - } - - private void verifyAuthorized(EntityDto entity, UserDto user, GroupDto group) { - logIn(user).setGroups(group); - verifyAuthorized(entity, true); - } - - private void verifyNotAuthorized(EntityDto entity, UserDto user) { - logIn(user); - verifyAuthorized(entity, false); - } - - private void verifyNotAuthorized(EntityDto entity, UserDto user, GroupDto group) { - logIn(user).setGroups(group); - verifyAuthorized(entity, false); - } - - private void verifyAnyoneAuthorized(EntityDto entity) { - userSession.anonymous(); - verifyAuthorized(entity, true); - } - - private void verifyAnyoneNotAuthorized(EntityDto entity) { - userSession.anonymous(); - verifyAuthorized(entity, false); - } - - private void verifyAuthorized(EntityDto entity, boolean expectedAccess) { - assertThat(fooIndex.hasAccessToProject(entity.getUuid())).isEqualTo(expectedAccess); - } - - private UserSessionRule logIn(UserDto u) { - userSession.logIn(u); - return userSession; - } - - private IndexingResult indexPermissions(EntityDto entity, EntityEvent cause) { - DbSession dbSession = db.getSession(); - Collection<EsQueueDto> items = underTest.prepareForRecoveryOnEntityEvent(dbSession, singletonList(entity.getUuid()), cause); - dbSession.commit(); - return underTest.index(dbSession, items); - } - - private ProjectDto createUnindexedPublicProject() { - return db.components().insertPublicProject().getProjectDto(); - } - - private ProjectDto createAndIndexPrivateProject() { - ProjectData project = db.components().insertPrivateProject(); - fooIndexer.indexOnAnalysis(project.getMainBranchDto().getUuid()); - return project.getProjectDto(); - } - - private ProjectDto createAndIndexPublicProject() { - ProjectData project = db.components().insertPublicProject(); - fooIndexer.indexOnAnalysis(project.getMainBranchDto().getUuid()); - return project.getProjectDto(); - } - - private PortfolioDto createAndIndexPortfolio() { - PortfolioDto view = db.components().insertPublicPortfolioDto(); - fooIndexer.indexOnAnalysis(view.getUuid()); - return view; - } - - private IndexingResult recover() { - Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), System.currentTimeMillis() + 1_000L, 10); - return underTest.index(db.getSession(), items); - } - - private void assertThatEsQueueTableHasSize(int expectedSize) { - assertThat(db.countRowsOfTable("es_queue")).isEqualTo(expectedSize); - } - -} diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java b/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndex.java index ab0ff8755eb..ab0ff8755eb 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java +++ b/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndex.java diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java b/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndexer.java index cd8e1c4c262..cd8e1c4c262 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java +++ b/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndexer.java |