From cad73b7391ef4d4d483b521eef9a94304e30c08e Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lievremont Date: Mon, 4 May 2015 18:03:36 +0200 Subject: [PATCH] SONAR-6521 Add internal WS to list user details for batch --- .../org/sonar/server/batch/BatchAction.java | 27 ++++++ .../java/org/sonar/server/batch/BatchWs.java | 16 ++-- .../server/batch/GlobalRepositoryAction.java | 6 +- .../org/sonar/server/batch/IssuesAction.java | 6 +- .../server/batch/ProjectRepositoryAction.java | 6 +- .../org/sonar/server/batch/UsersAction.java | 90 +++++++++++++++++++ .../server/platform/ServerComponents.java | 2 + .../sonar/server/user/index/UserIndex.java | 64 +++++++++++++ .../batch/GlobalRepositoryActionTest.java | 2 +- .../sonar/server/batch/IssuesActionTest.java | 15 ++-- .../batch/ProjectRepositoryActionTest.java | 3 +- .../sonar/server/batch/UsersActionTest.java | 85 ++++++++++++++++++ 12 files changed, 292 insertions(+), 30 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/batch/BatchAction.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/batch/UsersAction.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/batch/UsersActionTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/BatchAction.java b/server/sonar-server/src/main/java/org/sonar/server/batch/BatchAction.java new file mode 100644 index 00000000000..367935d1ade --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/BatchAction.java @@ -0,0 +1,27 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.batch; + +import org.sonar.server.ws.WsAction; + +public interface BatchAction extends WsAction { + + // Marker interface +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/BatchWs.java b/server/sonar-server/src/main/java/org/sonar/server/batch/BatchWs.java index fdddd299e55..383c1847cf9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/batch/BatchWs.java +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/BatchWs.java @@ -33,15 +33,11 @@ public class BatchWs implements WebService { public static final String API_ENDPOINT = "batch"; private final BatchIndex batchIndex; - private final GlobalRepositoryAction globalRepositoryAction; - private final ProjectRepositoryAction projectRepositoryAction; - private final IssuesAction issuesAction; + private final BatchAction[] actions; - public BatchWs(BatchIndex batchIndex, GlobalRepositoryAction globalRepositoryAction, ProjectRepositoryAction projectRepositoryAction, IssuesAction issuesAction) { + public BatchWs(BatchIndex batchIndex, BatchAction... actions) { this.batchIndex = batchIndex; - this.globalRepositoryAction = globalRepositoryAction; - this.projectRepositoryAction = projectRepositoryAction; - this.issuesAction = issuesAction; + this.actions = actions; } @Override @@ -52,9 +48,9 @@ public class BatchWs implements WebService { defineIndexAction(controller); defineFileAction(controller); - globalRepositoryAction.define(controller); - projectRepositoryAction.define(controller); - issuesAction.define(controller); + for (BatchAction action : actions) { + action.define(controller); + } controller.done(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/GlobalRepositoryAction.java b/server/sonar-server/src/main/java/org/sonar/server/batch/GlobalRepositoryAction.java index ae6a9a06d16..0e19a80fce6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/batch/GlobalRepositoryAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/GlobalRepositoryAction.java @@ -22,7 +22,6 @@ package org.sonar.server.batch; import org.apache.commons.io.IOUtils; import org.sonar.api.server.ws.Request; -import org.sonar.api.server.ws.RequestHandler; import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; import org.sonar.batch.protocol.input.GlobalRepositories; @@ -37,7 +36,7 @@ import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.plugins.MimeTypes; import org.sonar.server.user.UserSession; -public class GlobalRepositoryAction implements RequestHandler { +public class GlobalRepositoryAction implements BatchAction { private final DbClient dbClient; private final PropertiesDao propertiesDao; @@ -47,7 +46,8 @@ public class GlobalRepositoryAction implements RequestHandler { this.propertiesDao = propertiesDao; } - void define(WebService.NewController controller) { + @Override + public void define(WebService.NewController controller) { controller.createAction("global") .setDescription("Return metrics and global properties") .setSince("4.5") diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/IssuesAction.java b/server/sonar-server/src/main/java/org/sonar/server/batch/IssuesAction.java index c19c985787e..a0e9bf40c67 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/batch/IssuesAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/IssuesAction.java @@ -22,7 +22,6 @@ package org.sonar.server.batch; import org.sonar.api.resources.Scopes; import org.sonar.api.server.ws.Request; -import org.sonar.api.server.ws.RequestHandler; import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; import org.sonar.batch.protocol.input.BatchInput; @@ -44,7 +43,7 @@ import java.util.Map; import static com.google.common.collect.Maps.newHashMap; -public class IssuesAction implements RequestHandler { +public class IssuesAction implements BatchAction { private static final String PARAM_KEY = "key"; @@ -57,7 +56,8 @@ public class IssuesAction implements RequestHandler { this.issueIndex = issueIndex; } - void define(WebService.NewController controller) { + @Override + public void define(WebService.NewController controller) { WebService.NewAction action = controller.createAction("issues") .setDescription("Return open issues") .setSince("5.1") diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryAction.java b/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryAction.java index baa4e4a703d..9b7557aafdf 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/ProjectRepositoryAction.java @@ -22,13 +22,12 @@ package org.sonar.server.batch; import org.apache.commons.io.IOUtils; import org.sonar.api.server.ws.Request; -import org.sonar.api.server.ws.RequestHandler; import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; import org.sonar.batch.protocol.input.ProjectRepositories; import org.sonar.server.plugins.MimeTypes; -public class ProjectRepositoryAction implements RequestHandler { +public class ProjectRepositoryAction implements BatchAction { private static final String PARAM_KEY = "key"; private static final String PARAM_PROFILE = "profile"; @@ -40,7 +39,8 @@ public class ProjectRepositoryAction implements RequestHandler { this.projectReferentialsLoader = projectReferentialsLoader; } - void define(WebService.NewController controller) { + @Override + public void define(WebService.NewController controller) { WebService.NewAction action = controller.createAction("project") .setDescription("Return project repository") .setSince("4.5") diff --git a/server/sonar-server/src/main/java/org/sonar/server/batch/UsersAction.java b/server/sonar-server/src/main/java/org/sonar/server/batch/UsersAction.java new file mode 100644 index 00000000000..520ce7e6757 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/batch/UsersAction.java @@ -0,0 +1,90 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.batch; + +import org.sonar.api.server.ws.Request; +import org.sonar.api.server.ws.Response; +import org.sonar.api.server.ws.WebService; +import org.sonar.batch.protocol.input.BatchInput; +import org.sonar.core.permission.GlobalPermissions; +import org.sonar.server.plugins.MimeTypes; +import org.sonar.server.user.UserSession; +import org.sonar.server.user.index.UserDoc; +import org.sonar.server.user.index.UserIndex; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Iterator; +import java.util.List; + +public class UsersAction implements BatchAction { + + private static final String PARAM_LOGINS = "logins"; + + private final UserIndex userIndex; + + public UsersAction(UserIndex userIndex) { + this.userIndex = userIndex; + } + + @Override + public void define(WebService.NewController controller) { + WebService.NewAction action = controller.createAction("users") + .setDescription("Return user details.") + .setSince("5.2") + .setInternal(true) + .setHandler(this); + + action + .createParam(PARAM_LOGINS) + .setRequired(true) + .setDescription("A comma separated list of user logins") + .setExampleValue("ada.lovelace,grace.hopper"); + } + + @Override + public void handle(Request request, Response response) throws Exception { + UserSession.get().checkGlobalPermission(GlobalPermissions.PREVIEW_EXECUTION); + List logins = request.mandatoryParamAsStrings(PARAM_LOGINS); + + response.stream().setMediaType(MimeTypes.PROTOBUF); + BatchInput.User.Builder userBuilder = BatchInput.User.newBuilder(); + OutputStream output = response.stream().output(); + try { + for (Iterator userDocIterator = userIndex.selectUsersForBatch(logins); userDocIterator.hasNext();) { + handleUser(userDocIterator.next(), userBuilder, output); + } + } finally { + output.close(); + } + } + + private void handleUser(UserDoc user, BatchInput.User.Builder userBuilder, OutputStream out) { + userBuilder.setLogin(user.login()) + .setName(user.name()); + try { + userBuilder.build().writeDelimitedTo(out); + } catch (IOException e) { + throw new IllegalStateException("Unable to serialize user", e); + } + userBuilder.clear(); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index 2ceda78a3d4..67679ebd960 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -95,6 +95,7 @@ import org.sonar.server.batch.GlobalRepositoryAction; import org.sonar.server.batch.IssuesAction; import org.sonar.server.batch.ProjectRepositoryAction; import org.sonar.server.batch.ProjectRepositoryLoader; +import org.sonar.server.batch.UsersAction; import org.sonar.server.charts.ChartFactory; import org.sonar.server.component.ComponentCleanerService; import org.sonar.server.component.ComponentService; @@ -611,6 +612,7 @@ class ServerComponents { pico.addSingleton(ProjectRepositoryLoader.class); pico.addSingleton(SubmitReportWsAction.class); pico.addSingleton(IssuesAction.class); + pico.addSingleton(UsersAction.class); pico.addSingleton(BatchWs.class); // Dashboard diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java index 73a241e3f68..791c6642e88 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java @@ -24,20 +24,34 @@ import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.BoolFilterBuilder; import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.ServerComponent; import org.sonar.server.es.EsClient; import org.sonar.server.exceptions.NotFoundException; import javax.annotation.CheckForNull; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; public class UserIndex implements ServerComponent { + private static final int SCROLL_TIME_IN_MINUTES = 3; + private final EsClient esClient; public UserIndex(EsClient esClient) { @@ -100,4 +114,54 @@ public class UserIndex implements ServerComponent { return result; } + public Iterator selectUsersForBatch(List logins) { + BoolFilterBuilder filter = FilterBuilders.boolFilter() + .must(FilterBuilders.termsFilter(UserIndexDefinition.FIELD_LOGIN, logins)); + + SearchRequestBuilder requestBuilder = esClient + .prepareSearch(UserIndexDefinition.INDEX) + .setTypes(UserIndexDefinition.TYPE_USER) + .setSearchType(SearchType.SCAN) + .addSort(SortBuilders.fieldSort(UserIndexDefinition.FIELD_LOGIN).order(SortOrder.ASC)) + .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) + .setSize(10000) + .setFetchSource( + new String[] {UserIndexDefinition.FIELD_LOGIN, UserIndexDefinition.FIELD_NAME}, + null) + .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter)); + SearchResponse response = requestBuilder.get(); + + return scroll(response.getScrollId()); + } + + // Scrolling within the index + private Iterator scroll(final String scrollId) { + return new Iterator() { + private final Queue hits = new ArrayDeque<>(); + + @Override + public boolean hasNext() { + if (hits.isEmpty()) { + SearchScrollRequestBuilder esRequest = esClient.prepareSearchScroll(scrollId) + .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + Collections.addAll(hits, esRequest.get().getHits().getHits()); + } + return !hits.isEmpty(); + } + + @Override + public UserDoc next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return new UserDoc(hits.poll().getSource()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Cannot remove item when scrolling"); + } + }; + } + } diff --git a/server/sonar-server/src/test/java/org/sonar/server/batch/GlobalRepositoryActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/batch/GlobalRepositoryActionTest.java index 5afe8f54be7..6548bfe6bf7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/batch/GlobalRepositoryActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/batch/GlobalRepositoryActionTest.java @@ -65,7 +65,7 @@ public class GlobalRepositoryActionTest { when(dbClient.openSession(false)).thenReturn(session); when(dbClient.metricDao()).thenReturn(metricDao); - tester = new WsTester(new BatchWs(mock(BatchIndex.class), new GlobalRepositoryAction(dbClient, propertiesDao), mock(ProjectRepositoryAction.class), mock(IssuesAction.class))); + tester = new WsTester(new BatchWs(mock(BatchIndex.class), new GlobalRepositoryAction(dbClient, propertiesDao))); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java index 9186253b320..efb7729e583 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java @@ -35,7 +35,6 @@ import org.sonar.core.component.ComponentDto; import org.sonar.core.permission.GlobalPermissions; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.DbTester; -import org.sonar.core.properties.PropertiesDao; import org.sonar.server.component.ComponentTesting; import org.sonar.server.component.db.ComponentDao; import org.sonar.server.db.DbClient; @@ -43,7 +42,12 @@ import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.issue.IssueTesting; import org.sonar.server.issue.db.IssueDao; -import org.sonar.server.issue.index.*; +import org.sonar.server.issue.index.IssueAuthorizationDao; +import org.sonar.server.issue.index.IssueAuthorizationIndexer; +import org.sonar.server.issue.index.IssueDoc; +import org.sonar.server.issue.index.IssueIndex; +import org.sonar.server.issue.index.IssueIndexDefinition; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.user.MockUserSession; import org.sonar.server.ws.WsTester; import org.sonar.test.DbTests; @@ -94,12 +98,7 @@ public class IssuesActionTest { issuesAction = new IssuesAction(dbClient, issueIndex); componentDao = new ComponentDao(); - tester = new WsTester(new BatchWs( - new BatchIndex(mock(Server.class)), - new GlobalRepositoryAction(mock(DbClient.class), mock(PropertiesDao.class)), - new ProjectRepositoryAction(mock(ProjectRepositoryLoader.class)), - issuesAction) - ); + tester = new WsTester(new BatchWs(new BatchIndex(mock(Server.class)), issuesAction)); } @After diff --git a/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryActionTest.java index 7085f4b2499..806196fb2f9 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryActionTest.java @@ -43,8 +43,7 @@ public class ProjectRepositoryActionTest { @Before public void setUp() throws Exception { - tester = new WsTester(new BatchWs(mock(BatchIndex.class), mock(GlobalRepositoryAction.class), - new ProjectRepositoryAction(projectRepositoryLoader), mock(IssuesAction.class))); + tester = new WsTester(new BatchWs(mock(BatchIndex.class), new ProjectRepositoryAction(projectRepositoryLoader))); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/batch/UsersActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/batch/UsersActionTest.java new file mode 100644 index 00000000000..1daa51c3dff --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/batch/UsersActionTest.java @@ -0,0 +1,85 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.batch; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.api.platform.Server; +import org.sonar.batch.protocol.input.BatchInput.User; +import org.sonar.core.permission.GlobalPermissions; +import org.sonar.server.es.EsTester; +import org.sonar.server.user.MockUserSession; +import org.sonar.server.user.index.UserDoc; +import org.sonar.server.user.index.UserIndex; +import org.sonar.server.user.index.UserIndexDefinition; +import org.sonar.server.ws.WsTester; + +import java.io.ByteArrayInputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +public class UsersActionTest { + + @ClassRule + public static EsTester es = new EsTester().addDefinitions(new UserIndexDefinition(new Settings())); + + UserIndex userIndex; + + WsTester tester; + + UsersAction usersAction; + + @Before + public void before() throws Exception { + es.truncateIndices(); + + userIndex = new UserIndex(es.client()); + usersAction = new UsersAction(userIndex); + + tester = new WsTester(new BatchWs(new BatchIndex(mock(Server.class)), usersAction)); + } + + @Test + public void return_minimal_fields() throws Exception { + es.putDocuments(UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, + new UserDoc().setLogin("ada.lovelace").setName("Ada Lovelace").setActive(false), + new UserDoc().setLogin("grace.hopper").setName("Grace Hopper").setActive(true)); + + MockUserSession.set().setLogin("sonarqtech").setGlobalPermissions(GlobalPermissions.PREVIEW_EXECUTION); + + WsTester.TestRequest request = tester.newGetRequest("batch", "users").setParam("logins", "ada.lovelace,grace.hopper"); + + ByteArrayInputStream input = new ByteArrayInputStream(request.execute().output()); + + User user = User.parseDelimitedFrom(input); + assertThat(user.getLogin()).isEqualTo("ada.lovelace"); + assertThat(user.getName()).isEqualTo("Ada Lovelace"); + + user = User.parseDelimitedFrom(input); + assertThat(user.getLogin()).isEqualTo("grace.hopper"); + assertThat(user.getName()).isEqualTo("Grace Hopper"); + + assertThat(User.parseDelimitedFrom(input)).isNull(); + } +} -- 2.39.5