From 68de595dc688e9fd09bd8925c94d0bba830c3869 Mon Sep 17 00:00:00 2001 From: Aurelien Poscia Date: Thu, 30 Nov 2023 09:58:20 +0100 Subject: [PATCH] SONAR-21119 Provide method to get groups for GitLab & refactored GithubPaginatedHttpClient and GithubApplicationHttpClient to make them generic --- server/sonar-alm-client/build.gradle | 1 + .../{github => }/ApplicationHttpClient.java | 2 +- .../{github => }/DevopsPlatformHeaders.java | 2 +- .../GenericApplicationHttpClient.java | 22 ++-- .../client/GenericPaginatedHttpClient.java | 98 ++++++++++++++++ .../{github => }/PaginatedHttpClient.java | 6 +- .../RatioBasedRateLimitChecker.java | 6 +- .../github/GithubApplicationClientImpl.java | 53 +++------ .../github/GithubApplicationHttpClient.java | 1 + .../alm/client/github/GithubHeaders.java | 1 + .../github/GithubPaginatedHttpClient.java | 62 +--------- ...ient.java => GitlabApplicationClient.java} | 54 ++++----- .../gitlab/GitlabApplicationHttpClient.java | 33 ++++++ .../gitlab/GitlabGlobalSettingsValidator.java | 14 +-- .../alm/client/gitlab/GitlabHeaders.java | 60 ++++++++++ .../gitlab/GitlabPaginatedHttpClient.java | 35 ++++++ .../sonar/alm/client/gitlab/GitlabToken.java | 58 ++++++++++ ...> GenericPaginatedHttpClientImplTest.java} | 34 +++++- .../RatioBasedRateLimitCheckerTest.java | 4 +- .../GenericApplicationHttpClientTest.java | 55 ++++++--- .../GithubApplicationClientImplTest.java | 109 ++++++++---------- ....java => GitlabApplicationClientTest.java} | 69 ++++++++++- .../GitlabGlobalSettingsValidatorTest.java | 4 +- .../client/gitlab/groups-full-response.json | 76 ++++++++++++ .../java/org/sonar/auth/gitlab/GsonGroup.java | 20 +++- .../org/sonar/auth/gitlab/GsonGroupTest.java | 4 +- .../almintegration/ws/CheckPatActionIT.java | 10 +- .../gitlab/ImportGitLabProjectActionIT.java | 22 ++-- .../ws/gitlab/SearchGitlabReposActionIT.java | 14 +-- .../almintegration/ws/CheckPatAction.java | 10 +- .../ws/gitlab/ImportGitLabProjectAction.java | 12 +- .../ws/gitlab/SearchGitlabReposAction.java | 10 +- .../platformlevel/PlatformLevel4.java | 16 ++- 33 files changed, 683 insertions(+), 294 deletions(-) rename server/sonar-alm-client/src/main/java/org/sonar/alm/client/{github => }/ApplicationHttpClient.java (98%) rename server/sonar-alm-client/src/main/java/org/sonar/alm/client/{github => }/DevopsPlatformHeaders.java (96%) rename server/sonar-alm-client/src/main/java/org/sonar/alm/client/{github => }/GenericApplicationHttpClient.java (95%) create mode 100644 server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericPaginatedHttpClient.java rename server/sonar-alm-client/src/main/java/org/sonar/alm/client/{github => }/PaginatedHttpClient.java (88%) rename server/sonar-alm-client/src/main/java/org/sonar/alm/client/{github => }/RatioBasedRateLimitChecker.java (88%) rename server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/{GitlabHttpClient.java => GitlabApplicationClient.java} (90%) create mode 100644 server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationHttpClient.java create mode 100644 server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHeaders.java create mode 100644 server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabPaginatedHttpClient.java create mode 100644 server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabToken.java rename server/sonar-alm-client/src/test/java/org/sonar/alm/client/{github/GithubPaginatedHttpClientImplTest.java => GenericPaginatedHttpClientImplTest.java} (80%) rename server/sonar-alm-client/src/test/java/org/sonar/alm/client/{github => }/RatioBasedRateLimitCheckerTest.java (95%) rename server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/{GitlabHttpClientTest.java => GitlabApplicationClientTest.java} (88%) create mode 100644 server/sonar-alm-client/src/test/resources/org/sonar/alm/client/gitlab/groups-full-response.json diff --git a/server/sonar-alm-client/build.gradle b/server/sonar-alm-client/build.gradle index 7f77befefca..6c180f5605e 100644 --- a/server/sonar-alm-client/build.gradle +++ b/server/sonar-alm-client/build.gradle @@ -13,6 +13,7 @@ dependencies { api 'org.bouncycastle:bcpkix-jdk18on:1.76' api 'org.sonarsource.api.plugin:sonar-plugin-api' api project(':server:sonar-auth-github') + api project(':server:sonar-auth-gitlab') testImplementation project(':sonar-plugin-api-impl') diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/ApplicationHttpClient.java similarity index 98% rename from server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java rename to server/sonar-alm-client/src/main/java/org/sonar/alm/client/ApplicationHttpClient.java index 75c512eddb5..933af0910f1 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/ApplicationHttpClient.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import java.io.IOException; import java.util.Optional; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/DevopsPlatformHeaders.java similarity index 96% rename from server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java rename to server/sonar-alm-client/src/main/java/org/sonar/alm/client/DevopsPlatformHeaders.java index 38e8fe0b94c..210a562b260 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/DevopsPlatformHeaders.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import java.util.Optional; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericApplicationHttpClient.java similarity index 95% rename from server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java rename to server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericApplicationHttpClient.java index d67224c2615..fc375f96992 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericApplicationHttpClient.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import java.io.IOException; import java.net.MalformedURLException; @@ -37,7 +37,6 @@ import okhttp3.ResponseBody; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.alm.client.github.security.AccessToken; import org.sonarqube.ws.client.OkHttpClientBuilder; @@ -58,7 +57,7 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli private final DevopsPlatformHeaders devopsPlatformHeaders; private final OkHttpClient client; - public GenericApplicationHttpClient(DevopsPlatformHeaders devopsPlatformHeaders, TimeoutConfiguration timeoutConfiguration) { + protected GenericApplicationHttpClient(DevopsPlatformHeaders devopsPlatformHeaders, TimeoutConfiguration timeoutConfiguration) { this.devopsPlatformHeaders = devopsPlatformHeaders; client = new OkHttpClientBuilder() .setConnectTimeoutMs(timeoutConfiguration.getConnectTimeout()) @@ -184,7 +183,7 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli private Request.Builder newRequestBuilder(String appUrl, @Nullable AccessToken token, String endPoint) { Request.Builder url = new Request.Builder().url(toAbsoluteEndPoint(appUrl, endPoint)); if (token != null) { - url.addHeader(devopsPlatformHeaders.getAuthorizationHeader(), token.getAuthorizationHeaderPrefix() + " " + token); + url.addHeader(devopsPlatformHeaders.getAuthorizationHeader(), token.getAuthorizationHeaderPrefix() + " " + token.getValue()); devopsPlatformHeaders.getApiVersion().ifPresent(apiVersion -> url.addHeader(devopsPlatformHeaders.getApiVersionHeader().orElseThrow(), apiVersion) ); @@ -224,17 +223,16 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli @CheckForNull private static String readNextEndPoint(okhttp3.Response response) { - String links = response.headers().get("link"); - if (links == null || links.isEmpty() || !links.contains("rel=\"next\"")) { - return null; - } - + String links = Optional.ofNullable(response.headers().get("link")).orElse(""); Matcher nextLinkMatcher = NEXT_LINK_PATTERN.matcher(links); if (!nextLinkMatcher.find()) { return null; } - - return nextLinkMatcher.group(1); + String nextUrl = nextLinkMatcher.group(1); + if (response.request().url().toString().equals(nextUrl)) { + return null; + } + return nextUrl; } @CheckForNull @@ -250,7 +248,7 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli @CheckForNull private static T headerValueOrNull(okhttp3.Response response, String header, Function mapper) { - return ofNullable(response.header(header)).map(mapper::apply).orElse(null); + return ofNullable(response.headers().get(header)).map(mapper::apply).orElse(null); } private static class ResponseImpl implements Response { diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericPaginatedHttpClient.java new file mode 100644 index 00000000000..51e948318ef --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericPaginatedHttpClient.java @@ -0,0 +1,98 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.alm.client.ApplicationHttpClient.GetResponse; +import org.sonar.alm.client.github.security.AccessToken; + +import static java.lang.String.format; + +public abstract class GenericPaginatedHttpClient implements PaginatedHttpClient { + + private static final Logger LOG = LoggerFactory.getLogger(GenericPaginatedHttpClient.class); + private final ApplicationHttpClient appHttpClient; + private final RatioBasedRateLimitChecker rateLimitChecker; + + protected GenericPaginatedHttpClient(ApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + this.appHttpClient = appHttpClient; + this.rateLimitChecker = rateLimitChecker; + } + + @Override + public List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) { + List results = new ArrayList<>(); + String nextEndpoint = query + "?per_page=100"; + if (query.contains("?")) { + nextEndpoint = query + "&per_page=100"; + } + ApplicationHttpClient.RateLimit rateLimit = null; + while (nextEndpoint != null) { + checkRateLimit(rateLimit); + GetResponse response = executeCall(appUrl, token, nextEndpoint); + response.getContent() + .ifPresent(content -> results.addAll(responseDeserializer.apply(content))); + nextEndpoint = response.getNextEndPoint().orElse(null); + rateLimit = response.getRateLimit(); + } + return results; + } + + private void checkRateLimit(@Nullable ApplicationHttpClient.RateLimit rateLimit) { + if (rateLimit == null) { + return; + } + try { + rateLimitChecker.checkRateLimit(rateLimit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn(format("Thread interrupted: %s", e.getMessage()), e); + } + } + + private GetResponse executeCall(String appUrl, AccessToken token, String endpoint) { + try { + GetResponse response = appHttpClient.get(appUrl, token, endpoint); + if (response.getCode() < 200 || response.getCode() >= 300) { + throw new IllegalStateException( + format("Error while executing a call to %s. Return code %s. Error message: %s.", appUrl, response.getCode(), response.getContent().orElse(""))); + } + return response; + } catch (Exception e) { + String errorMessage = format("SonarQube was not able to retrieve resources from external system. Error while executing a paginated call to %s, endpoint:%s.", + appUrl, endpoint); + logException(errorMessage, e); + throw new IllegalStateException(errorMessage + " " + e.getMessage()); + } + } + + private static void logException(String message, Exception e) { + if (LOG.isDebugEnabled()) { + LOG.warn(message, e); + } else { + LOG.warn(message, e.getMessage()); + } + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/PaginatedHttpClient.java similarity index 88% rename from server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java rename to server/sonar-alm-client/src/main/java/org/sonar/alm/client/PaginatedHttpClient.java index 134e942671e..5e746e5709a 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/PaginatedHttpClient.java @@ -17,14 +17,12 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; -import java.io.IOException; import java.util.List; import java.util.function.Function; import org.sonar.alm.client.github.security.AccessToken; public interface PaginatedHttpClient { - - List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException; + List get(String appUrl, AccessToken token, String query, Function> responseDeserializer); } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/RatioBasedRateLimitChecker.java similarity index 88% rename from server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java rename to server/sonar-alm-client/src/main/java/org/sonar/alm/client/RatioBasedRateLimitChecker.java index 9eb6e46e493..01beeac4489 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/RatioBasedRateLimitChecker.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import com.google.common.annotations.VisibleForTesting; import org.kohsuke.github.GHRateLimit; @@ -33,7 +33,7 @@ public class RatioBasedRateLimitChecker extends RateLimitChecker { private static final Logger LOGGER = LoggerFactory.getLogger(RatioBasedRateLimitChecker.class); @VisibleForTesting - static final String RATE_RATIO_EXCEEDED_MESSAGE = "The GitHub API rate limit is almost reached. Pausing GitHub provisioning until the next rate limit reset. " + static final String RATE_RATIO_EXCEEDED_MESSAGE = "The external system API rate limit is almost reached. Pausing GitHub provisioning until the next rate limit reset. " + "{} out of {} calls were used."; private static final int MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING = 90; @@ -42,7 +42,7 @@ public class RatioBasedRateLimitChecker extends RateLimitChecker { int limit = rateLimitRecord.limit(); int apiCallsUsed = limit - rateLimitRecord.remaining(); double percentageOfCallsUsed = computePercentageOfCallsUsed(apiCallsUsed, limit); - LOGGER.debug("{} GitHub API calls used of {} available per hours", apiCallsUsed, limit); + LOGGER.debug("{} external system API calls used of {}", apiCallsUsed, limit); if (percentageOfCallsUsed >= MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING) { LOGGER.warn(RATE_RATIO_EXCEEDED_MESSAGE, apiCallsUsed, limit); GHRateLimit.Record rateLimit = new GHRateLimit.Record(rateLimitRecord.limit(), rateLimitRecord.remaining(), rateLimitRecord.reset()); diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java index 2e657f7f3f4..d079dc6fb27 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java @@ -37,7 +37,8 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; +import org.sonar.alm.client.ApplicationHttpClient; +import org.sonar.alm.client.ApplicationHttpClient.GetResponse; import org.sonar.alm.client.github.GithubBinding.GsonGithubRepository; import org.sonar.alm.client.github.GithubBinding.GsonInstallations; import org.sonar.alm.client.github.GithubBinding.GsonRepositorySearch; @@ -69,21 +70,17 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { protected static final String WRITE_PERMISSION_NAME = "write"; protected static final String READ_PERMISSION_NAME = "read"; protected static final String FAILED_TO_REQUEST_BEGIN_MSG = "Failed to request "; - - private static final String EXCEPTION_MESSAGE = "SonarQube was not able to retrieve resources from GitHub. " - + "This is likely due to a connectivity problem or a temporary network outage"; - private static final Type REPOSITORY_TEAM_LIST_TYPE = TypeToken.getParameterized(List.class, GsonRepositoryTeam.class).getType(); private static final Type REPOSITORY_COLLABORATORS_LIST_TYPE = TypeToken.getParameterized(List.class, GsonRepositoryCollaborator.class).getType(); private static final Type ORGANIZATION_LIST_TYPE = TypeToken.getParameterized(List.class, GithubBinding.GsonInstallation.class).getType(); - protected final ApplicationHttpClient appHttpClient; + protected final GithubApplicationHttpClient githubApplicationHttpClient; protected final GithubAppSecurity appSecurity; private final GitHubSettings gitHubSettings; - private final PaginatedHttpClient githubPaginatedHttpClient; + private final GithubPaginatedHttpClient githubPaginatedHttpClient; - public GithubApplicationClientImpl(ApplicationHttpClient appHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings, - PaginatedHttpClient githubPaginatedHttpClient) { - this.appHttpClient = appHttpClient; + public GithubApplicationClientImpl(GithubApplicationHttpClient githubApplicationHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings, + GithubPaginatedHttpClient githubPaginatedHttpClient) { + this.githubApplicationHttpClient = githubApplicationHttpClient; this.appSecurity = appSecurity; this.gitHubSettings = gitHubSettings; this.githubPaginatedHttpClient = githubPaginatedHttpClient; @@ -106,7 +103,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { private Optional post(String baseUrl, AccessToken token, String endPoint, Class gsonClass) { try { - ApplicationHttpClient.Response response = appHttpClient.post(baseUrl, token, endPoint); + ApplicationHttpClient.Response response = githubApplicationHttpClient.post(baseUrl, token, endPoint); return handleResponse(response, endPoint, gsonClass); } catch (Exception e) { LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + endPoint, e); @@ -146,7 +143,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { String endPoint = "/app"; GetResponse response; try { - response = appHttpClient.get(githubAppConfiguration.getApiEndpoint(), appToken, endPoint); + response = githubApplicationHttpClient.get(githubAppConfiguration.getApiEndpoint(), appToken, endPoint); } catch (IOException e) { LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + githubAppConfiguration.getApiEndpoint() + endPoint, e); throw new IllegalArgumentException("Failed to validate configuration, check URL and Private Key"); @@ -189,7 +186,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { try { Organizations organizations = new Organizations(); - GetResponse response = appHttpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", page, pageSize)); + GetResponse response = githubApplicationHttpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", page, pageSize)); Optional gsonInstallations = response.getContent().map(content -> GSON.fromJson(content, GsonInstallations.class)); if (!gsonInstallations.isPresent()) { @@ -247,7 +244,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { protected Optional get(String baseUrl, AccessToken token, String endPoint, Class gsonClass) { try { - GetResponse response = appHttpClient.get(baseUrl, token, endPoint); + GetResponse response = githubApplicationHttpClient.get(baseUrl, token, endPoint); return handleResponse(response, endPoint, gsonClass); } catch (Exception e) { LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + endPoint, e); @@ -264,7 +261,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { } try { Repositories repositories = new Repositories(); - GetResponse response = appHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", searchQuery, page, pageSize)); + GetResponse response = githubApplicationHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", searchQuery, page, pageSize)); Optional gsonRepositories = response.getContent().map(content -> GSON.fromJson(content, GsonRepositorySearch.class)); if (!gsonRepositories.isPresent()) { return repositories; @@ -288,7 +285,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { @Override public Optional getRepository(String appUrl, AccessToken accessToken, String organizationAndRepository) { try { - GetResponse response = appHttpClient.get(appUrl, accessToken, String.format("/repos/%s", organizationAndRepository)); + GetResponse response = githubApplicationHttpClient.get(appUrl, accessToken, String.format("/repos/%s", organizationAndRepository)); return Optional.of(response) .filter(r -> r.getCode() == HTTP_OK) .flatMap(ApplicationHttpClient.Response::getContent) @@ -315,7 +312,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { baseAppUrl = appUrl; } - ApplicationHttpClient.Response response = appHttpClient.post(baseAppUrl, null, endpoint); + ApplicationHttpClient.Response response = githubApplicationHttpClient.post(baseAppUrl, null, endpoint); if (response.getCode() != HTTP_OK) { throw new IllegalStateException("Failed to create GitHub's user access token. GitHub returned code " + code + ". " + response.getContent().orElse("")); @@ -333,7 +330,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { } // If token is not in the 200's body, it's because the client ID or client secret are incorrect - LOG.error("Failed to create GitHub's user access token. GitHub's response: " + content); + LOG.error("Failed to create GitHub's user access token. GitHub's response: {}", content); throw new IllegalArgumentException(); } catch (IOException e) { throw new IllegalStateException("Failed to create GitHub's user access token", e); @@ -349,7 +346,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { private T getOrThrowIfNotHttpOk(String baseUrl, AccessToken token, String endPoint, Class gsonClass) { try { - GetResponse response = appHttpClient.get(baseUrl, token, endPoint); + GetResponse response = githubApplicationHttpClient.get(baseUrl, token, endPoint); if (response.getCode() != HTTP_OK) { throw new HttpException(baseUrl + endPoint, response.getCode(), response.getContent().orElse("")); } @@ -386,23 +383,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { } private List executePaginatedQuery(String appUrl, AccessToken token, String query, Function> responseDeserializer) { - try { - return githubPaginatedHttpClient.get(appUrl, token, query, responseDeserializer); - } catch (IOException ioException) { - throw logAndCreateException(ioException, format("Error while executing a paginated call to GitHub - appUrl: %s, path: %s.", appUrl, query)); - } + return githubPaginatedHttpClient.get(appUrl, token, query, responseDeserializer); } - private static IllegalStateException logAndCreateException(IOException ioException, String errorMessage) { - log(errorMessage, ioException); - return new IllegalStateException(EXCEPTION_MESSAGE + ": " + errorMessage + " " + ioException.getMessage()); - } - - private static void log(String message, Exception e) { - if (LOG.isDebugEnabled()) { - LOG.warn(message, e); - } else { - LOG.warn(message); - } - } } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java index 49406cea9b0..24556e3da72 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java @@ -19,6 +19,7 @@ */ package org.sonar.alm.client.github; +import org.sonar.alm.client.GenericApplicationHttpClient; import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java index 9496f0b2bcf..847cf537507 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java @@ -20,6 +20,7 @@ package org.sonar.alm.client.github; import java.util.Optional; +import org.sonar.alm.client.DevopsPlatformHeaders; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java index 9c8d336e192..a68f0cf5d54 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java @@ -19,69 +19,17 @@ */ package org.sonar.alm.client.github; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.alm.client.GenericPaginatedHttpClient; +import org.sonar.alm.client.RatioBasedRateLimitChecker; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; -import static java.lang.String.format; - @ServerSide @ComputeEngineSide -public class GithubPaginatedHttpClient implements PaginatedHttpClient { - - private static final Logger LOG = LoggerFactory.getLogger(GithubPaginatedHttpClient.class); - private final ApplicationHttpClient appHttpClient; - private final RatioBasedRateLimitChecker rateLimitChecker; - - public GithubPaginatedHttpClient(ApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { - this.appHttpClient = appHttpClient; - this.rateLimitChecker = rateLimitChecker; - } +public class GithubPaginatedHttpClient extends GenericPaginatedHttpClient { - @Override - public List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException { - List results = new ArrayList<>(); - String nextEndpoint = query + "?per_page=100"; - if (query.contains("?")) { - nextEndpoint = query + "&per_page=100"; - } - ApplicationHttpClient.RateLimit rateLimit = null; - while (nextEndpoint != null) { - checkRateLimit(rateLimit); - ApplicationHttpClient.GetResponse response = executeCall(appUrl, token, nextEndpoint); - response.getContent() - .ifPresent(content -> results.addAll(responseDeserializer.apply(content))); - nextEndpoint = response.getNextEndPoint().orElse(null); - rateLimit = response.getRateLimit(); - } - return results; + public GithubPaginatedHttpClient(GithubApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + super(appHttpClient, rateLimitChecker); } - private void checkRateLimit(@Nullable ApplicationHttpClient.RateLimit rateLimit) { - if (rateLimit == null) { - return; - } - try { - rateLimitChecker.checkRateLimit(rateLimit); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn(format("Thread interrupted: %s", e.getMessage()), e); - } - } - - private ApplicationHttpClient.GetResponse executeCall(String appUrl, AccessToken token, String endpoint) throws IOException { - ApplicationHttpClient.GetResponse response = appHttpClient.get(appUrl, token, endpoint); - if (response.getCode() < 200 || response.getCode() >= 300) { - throw new IllegalStateException( - format("Error while executing a call to GitHub. Return code %s. Error message: %s.", response.getCode(), response.getContent().orElse(""))); - } - return response; - } } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationClient.java similarity index 90% rename from server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java rename to server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationClient.java index b93540c9057..13088aec6e7 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationClient.java @@ -19,15 +19,20 @@ */ package org.sonar.alm.client.gitlab; +import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Type; import java.net.URLEncoder; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import javax.annotation.Nullable; import okhttp3.Headers; import okhttp3.OkHttpClient; @@ -39,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.api.server.ServerSide; +import org.sonar.auth.gitlab.GsonGroup; import org.sonarqube.ws.MediaTypes; import org.sonarqube.ws.client.OkHttpClientBuilder; @@ -47,13 +53,18 @@ import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; import static java.nio.charset.StandardCharsets.UTF_8; @ServerSide -public class GitlabHttpClient { +public class GitlabApplicationClient { + private static final Logger LOG = LoggerFactory.getLogger(GitlabApplicationClient.class); + private static final Gson GSON = new Gson(); + private static final Type GITLAB_GROUP = TypeToken.getParameterized(List.class, GsonGroup.class).getType(); - private static final Logger LOG = LoggerFactory.getLogger(GitlabHttpClient.class); protected static final String PRIVATE_TOKEN = "Private-Token"; protected final OkHttpClient client; - public GitlabHttpClient(TimeoutConfiguration timeoutConfiguration) { + private final GitlabPaginatedHttpClient gitlabPaginatedHttpClient; + + public GitlabApplicationClient(GitlabPaginatedHttpClient gitlabPaginatedHttpClient, TimeoutConfiguration timeoutConfiguration) { + this.gitlabPaginatedHttpClient = gitlabPaginatedHttpClient; client = new OkHttpClientBuilder() .setConnectTimeoutMs(timeoutConfiguration.getConnectTimeout()) .setReadTimeoutMs(timeoutConfiguration.getReadTimeout()) @@ -324,34 +335,6 @@ public class GitlabHttpClient { } } - /*public void getGroups(String gitlabUrl, String token) { - String url = String.format("%s/groups", gitlabUrl); - LOG.debug(String.format("get groups : [%s]", url)); - - Request request = new Request.Builder() - .addHeader(PRIVATE_TOKEN, token) - .url(url) - .get() - .build(); - - - try (Response response = client.newCall(request).execute()) { - Headers headers = response.headers(); - checkResponseIsSuccessful(response, "Could not get projects from GitLab instance"); - List projectList = Project.parseJsonArray(response.body().string()); - int returnedPageNumber = parseAndGetIntegerHeader(headers.get("X-Page")); - int returnedPageSize = parseAndGetIntegerHeader(headers.get("X-Per-Page")); - String xtotal = headers.get("X-Total"); - Integer totalProjects = Strings.isEmpty(xtotal) ? null : parseAndGetIntegerHeader(xtotal); - return new ProjectList(projectList, returnedPageNumber, returnedPageSize, totalProjects); - } catch (JsonSyntaxException e) { - throw new IllegalArgumentException("Could not parse GitLab answer to search projects. Got a non-json payload as result."); - } catch (IOException e) { - logException(url, e); - throw new IllegalStateException(e.getMessage(), e); - } - }*/ - private static int parseAndGetIntegerHeader(@Nullable String header) { if (header == null) { throw new IllegalArgumentException("Pagination data from GitLab response is missing"); @@ -364,4 +347,13 @@ public class GitlabHttpClient { } } + public Set getGroups(String gitlabUrl, String token) { + return Set.copyOf(executePaginatedQuery(gitlabUrl, token, "/groups", resp -> GSON.fromJson(resp, GITLAB_GROUP))); + } + + private List executePaginatedQuery(String appUrl, String token, String query, Function> responseDeserializer) { + GitlabToken gitlabToken = new GitlabToken(token); + return gitlabPaginatedHttpClient.get(appUrl, gitlabToken, query, responseDeserializer); + } + } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationHttpClient.java new file mode 100644 index 00000000000..ec9c15f8673 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationHttpClient.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import org.sonar.alm.client.TimeoutConfiguration; +import org.sonar.alm.client.GenericApplicationHttpClient; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public class GitlabApplicationHttpClient extends GenericApplicationHttpClient { + public GitlabApplicationHttpClient(GitlabHeaders gitlabHeaders, TimeoutConfiguration timeoutConfiguration) { + super(gitlabHeaders, timeoutConfiguration); + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java index 2ca75b19e26..c1f76a15274 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java @@ -28,11 +28,11 @@ import org.sonar.db.alm.setting.AlmSettingDto; public class GitlabGlobalSettingsValidator { private final Encryption encryption; - private final GitlabHttpClient gitlabHttpClient; + private final GitlabApplicationClient gitlabApplicationClient; - public GitlabGlobalSettingsValidator(GitlabHttpClient gitlabHttpClient, Settings settings) { + public GitlabGlobalSettingsValidator(GitlabApplicationClient gitlabApplicationClient, Settings settings) { this.encryption = settings.getEncryption(); - this.gitlabHttpClient = gitlabHttpClient; + this.gitlabApplicationClient = gitlabApplicationClient; } public void validate(AlmSettingDto almSettingDto) { @@ -43,10 +43,10 @@ public class GitlabGlobalSettingsValidator { throw new IllegalArgumentException("Your Gitlab global configuration is incomplete."); } - gitlabHttpClient.checkUrl(gitlabUrl); - gitlabHttpClient.checkToken(gitlabUrl, accessToken); - gitlabHttpClient.checkReadPermission(gitlabUrl, accessToken); - gitlabHttpClient.checkWritePermission(gitlabUrl, accessToken); + gitlabApplicationClient.checkUrl(gitlabUrl); + gitlabApplicationClient.checkToken(gitlabUrl, accessToken); + gitlabApplicationClient.checkReadPermission(gitlabUrl, accessToken); + gitlabApplicationClient.checkWritePermission(gitlabUrl, accessToken); } } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHeaders.java new file mode 100644 index 00000000000..e09d7446e8a --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHeaders.java @@ -0,0 +1,60 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import java.util.Optional; +import org.sonar.alm.client.DevopsPlatformHeaders; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public class GitlabHeaders implements DevopsPlatformHeaders { + + @Override + public Optional getApiVersionHeader() { + return Optional.empty(); + } + + @Override + public Optional getApiVersion() { + return Optional.empty(); + } + + @Override + public String getRateLimitRemainingHeader() { + return "ratelimit-remaining"; + } + + @Override + public String getRateLimitLimitHeader() { + return "ratelimit-limit"; + } + + @Override + public String getRateLimitResetHeader() { + return "ratelimit-reset"; + } + + @Override + public String getAuthorizationHeader() { + return "PRIVATE-TOKEN"; + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabPaginatedHttpClient.java new file mode 100644 index 00000000000..0e02c84c53c --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabPaginatedHttpClient.java @@ -0,0 +1,35 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import org.sonar.alm.client.GenericPaginatedHttpClient; +import org.sonar.alm.client.RatioBasedRateLimitChecker; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public class GitlabPaginatedHttpClient extends GenericPaginatedHttpClient { + + public GitlabPaginatedHttpClient(GitlabApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + super(appHttpClient, rateLimitChecker); + } + +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabToken.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabToken.java new file mode 100644 index 00000000000..c6c3c0a3112 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabToken.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import java.util.Objects; +import org.sonar.alm.client.github.security.AccessToken; + +public class GitlabToken implements AccessToken { + private final String token; + + public GitlabToken(String token) { + this.token = token; + } + + @Override + public String getValue() { + return token; + } + + @Override + public String getAuthorizationHeaderPrefix() { + return ""; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + GitlabToken that = (GitlabToken) o; + return Objects.equals(token, that.token); + } + + @Override + public int hashCode() { + return Objects.hash(token); + } +} diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/GenericPaginatedHttpClientImplTest.java similarity index 80% rename from server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java rename to server/sonar-alm-client/src/test/java/org/sonar/alm/client/GenericPaginatedHttpClientImplTest.java index 5df514c857e..f9589290d53 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/GenericPaginatedHttpClientImplTest.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -45,10 +45,10 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; +import static org.sonar.alm.client.ApplicationHttpClient.GetResponse; @RunWith(MockitoJUnitRunner.class) -public class GithubPaginatedHttpClientImplTest { +public class GenericPaginatedHttpClientImplTest { private static final String APP_URL = "https://github.com/"; @@ -71,7 +71,13 @@ public class GithubPaginatedHttpClientImplTest { ApplicationHttpClient appHttpClient; @InjectMocks - private GithubPaginatedHttpClient underTest; + private TestPaginatedHttpClient underTest; + + private static class TestPaginatedHttpClient extends GenericPaginatedHttpClient { + protected TestPaginatedHttpClient(ApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + super(appHttpClient, rateLimitChecker); + } + } @Test public void get_whenNoPagination_ReturnsCorrectResponse() throws IOException { @@ -141,7 +147,8 @@ public class GithubPaginatedHttpClientImplTest { assertThatIllegalStateException() .isThrownBy(() -> underTest.get(APP_URL, accessToken, ENDPOINT, result -> gson.fromJson(result, STRING_LIST_TYPE))) - .withMessage("Error while executing a call to GitHub. Return code 400. Error message: failed."); + .withMessage("SonarQube was not able to retrieve resources from external system. Error while executing a paginated call to https://github.com/, endpoint:/next-endpoint. " + + "Error while executing a call to https://github.com/. Return code 400. Error message: failed."); } private static GetResponse mockFailedResponse(String content) { @@ -166,4 +173,21 @@ public class GithubPaginatedHttpClientImplTest { assertThat(logTester.logs(Level.WARN)) .containsExactly("Thread interrupted: interrupted"); } + + @Test + public void getRepositoryCollaborators_whenDevOpsPlatformCallThrowsIOException_shouldLogAndReThrow() throws IOException { + AccessToken accessToken = mock(); + when(appHttpClient.get(APP_URL, accessToken, "query?per_page=100")).thenThrow(new IOException("error")); + + assertThatIllegalStateException() + .isThrownBy(() -> underTest.get(APP_URL, accessToken, "query", mock())) + .isInstanceOf(IllegalStateException.class) + .withMessage("SonarQube was not able to retrieve resources from external system. Error while executing a paginated call to https://github.com/, " + + "endpoint:query?per_page=100. error"); + + assertThat(logTester.logs()).hasSize(1); + assertThat(logTester.logs(Level.WARN)) + .containsExactly("SonarQube was not able to retrieve resources from external system. " + + "Error while executing a paginated call to https://github.com/, endpoint:query?per_page=100."); + } } diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/RatioBasedRateLimitCheckerTest.java similarity index 95% rename from server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java rename to server/sonar-alm-client/src/test/java/org/sonar/alm/client/RatioBasedRateLimitCheckerTest.java index d10633365c6..d6d2bfff6e5 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/RatioBasedRateLimitCheckerTest.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; @@ -32,7 +32,7 @@ import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.sonar.alm.client.github.RatioBasedRateLimitChecker.RATE_RATIO_EXCEEDED_MESSAGE; +import static org.sonar.alm.client.RatioBasedRateLimitChecker.RATE_RATIO_EXCEEDED_MESSAGE; @RunWith(DataProviderRunner.class) public class RatioBasedRateLimitCheckerTest { diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GenericApplicationHttpClientTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GenericApplicationHttpClientTest.java index 53a6fcf08c9..dcd2fdab87c 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GenericApplicationHttpClientTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GenericApplicationHttpClientTest.java @@ -36,9 +36,11 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.slf4j.event.Level; import org.sonar.alm.client.ConstantTimeoutConfiguration; +import org.sonar.alm.client.DevopsPlatformHeaders; +import org.sonar.alm.client.GenericApplicationHttpClient; import org.sonar.alm.client.TimeoutConfiguration; -import org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; -import org.sonar.alm.client.github.ApplicationHttpClient.Response; +import org.sonar.alm.client.ApplicationHttpClient.GetResponse; +import org.sonar.alm.client.ApplicationHttpClient.Response; import org.sonar.alm.client.github.security.AccessToken; import org.sonar.alm.client.github.security.UserAccessToken; import org.sonar.api.testfixtures.log.LogTester; @@ -49,7 +51,7 @@ import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.Assert.fail; -import static org.sonar.alm.client.github.ApplicationHttpClient.RateLimit; +import static org.sonar.alm.client.ApplicationHttpClient.RateLimit; @RunWith(DataProviderRunner.class) public class GenericApplicationHttpClientTest { @@ -76,7 +78,7 @@ public class GenericApplicationHttpClientTest { logTester.clear(); } - private class TestApplicationHttpClient extends GenericApplicationHttpClient { + private static class TestApplicationHttpClient extends GenericApplicationHttpClient { public TestApplicationHttpClient(DevopsPlatformHeaders devopsPlatformHeaders, TimeoutConfiguration timeoutConfiguration) { super(devopsPlatformHeaders, timeoutConfiguration); } @@ -183,7 +185,7 @@ public class GenericApplicationHttpClientTest { public void get_returns_empty_endPoint_when_link_header_does_not_have_next_rel() throws IOException { server.enqueue(new MockResponse().setBody(randomBody) .setHeader("link", "; rel=\"prev\", " + - "; rel=\"first\"")); + "; rel=\"first\"")); GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); @@ -212,18 +214,30 @@ public class GenericApplicationHttpClientTest { assertThat(response.getNextEndPoint()).contains("https://api.github.com/installation/repositories?per_page=5&page=2"); } + @Test + public void get_returns_endPoint_when_link_header_is_from_gitlab() throws IOException { + String linkHeader = "; rel=\"next\", ; rel=\"first\", ; rel=\"last\""; + server.enqueue(new MockResponse().setBody(randomBody) + .setHeader("link", linkHeader)); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response.getNextEndPoint()).contains("https://gitlab.com/api/v4/groups?all_available=false" + + "&order_by=name&owned=false&page=2&per_page=2&sort=asc&statistics=false&with_custom_attributes=false"); + } + @DataProvider public static Object[][] linkHeadersWithNextRel() { String expected = "https://api.github.com/installation/repositories?per_page=5&page=2"; return new Object[][] { {"<" + expected + ">; rel=\"next\""}, {"<" + expected + ">; rel=\"next\", " + - "; rel=\"first\""}, + "; rel=\"first\""}, {"; rel=\"first\", " + - "<" + expected + ">; rel=\"next\""}, + "<" + expected + ">; rel=\"next\""}, {"; rel=\"first\", " + - "<" + expected + ">; rel=\"next\", " + - "; rel=\"last\""}, + "<" + expected + ">; rel=\"next\", " + + "; rel=\"last\""}, }; } @@ -416,14 +430,19 @@ public class GenericApplicationHttpClientTest { @Test public void get_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.get(appUrl, accessToken, randomEndPoint)); + testRateLimitHeader(() -> underTest.get(appUrl, accessToken, randomEndPoint), false); + } + + @Test + public void get_whenRateLimitHeadersArePresentAndUppercased_returnsRateLimit() throws Exception { + testRateLimitHeader(() -> underTest.get(appUrl, accessToken, randomEndPoint), true); } - private void testRateLimitHeader(Callable request ) throws Exception { + private void testRateLimitHeader(Callable request, boolean uppercasedHeaders) throws Exception { server.enqueue(new MockResponse().setBody(randomBody) - .setHeader("x-ratelimit-remaining", "1") - .setHeader("x-ratelimit-limit", "10") - .setHeader("x-ratelimit-reset", "1000")); + .setHeader(uppercasedHeaders ? "x-ratelimit-remaining" : "x-ratelimit-REMAINING", "1") + .setHeader(uppercasedHeaders ? "x-ratelimit-limit" : "X-RATELIMIT-LIMIT", "10") + .setHeader(uppercasedHeaders ? "x-ratelimit-reset" : "X-ratelimit-reset", "1000")); Response response = request.call(); @@ -438,7 +457,7 @@ public class GenericApplicationHttpClientTest { } - private void testMissingRateLimitHeader(Callable request ) throws Exception { + private void testMissingRateLimitHeader(Callable request) throws Exception { server.enqueue(new MockResponse().setBody(randomBody)); Response response = request.call(); @@ -448,7 +467,7 @@ public class GenericApplicationHttpClientTest { @Test public void delete_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.delete(appUrl, accessToken, randomEndPoint)); + testRateLimitHeader(() -> underTest.delete(appUrl, accessToken, randomEndPoint), false); } @@ -460,7 +479,7 @@ public class GenericApplicationHttpClientTest { @Test public void patch_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.patch(appUrl, accessToken, randomEndPoint, "body")); + testRateLimitHeader(() -> underTest.patch(appUrl, accessToken, randomEndPoint, "body"), false); } @Test @@ -470,7 +489,7 @@ public class GenericApplicationHttpClientTest { @Test public void post_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.post(appUrl, accessToken, randomEndPoint)); + testRateLimitHeader(() -> underTest.post(appUrl, accessToken, randomEndPoint), false); } @Test diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java index 6cf2f71dfc9..a7a3d2203d2 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java @@ -36,7 +36,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.slf4j.event.Level; -import org.sonar.alm.client.github.ApplicationHttpClient.RateLimit; +import org.sonar.alm.client.ApplicationHttpClient.RateLimit; import org.sonar.alm.client.github.api.GsonRepositoryCollaborator; import org.sonar.alm.client.github.api.GsonRepositoryTeam; import org.sonar.alm.client.github.config.GithubAppConfiguration; @@ -68,7 +68,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; +import static org.sonar.alm.client.ApplicationHttpClient.GetResponse; @RunWith(DataProviderRunner.class) public class GithubApplicationClientImplTest { @@ -114,12 +114,12 @@ public class GithubApplicationClientImplTest { @ClassRule public static LogTester logTester = new LogTester().setLevel(LoggerLevel.WARN); - private GenericApplicationHttpClient httpClient = mock(); + private GithubApplicationHttpClient githubApplicationHttpClient = mock(); private GithubAppSecurity appSecurity = mock(); private GithubAppConfiguration githubAppConfiguration = mock(); private GitHubSettings gitHubSettings = mock(); - private PaginatedHttpClient githubPaginatedHttpClient = mock(); + private GithubPaginatedHttpClient githubPaginatedHttpClient = mock(); private AppInstallationToken appInstallationToken = mock(); private GithubApplicationClient underTest; @@ -129,7 +129,7 @@ public class GithubApplicationClientImplTest { @Before public void setup() { when(githubAppConfiguration.getApiEndpoint()).thenReturn(appUrl); - underTest = new GithubApplicationClientImpl(httpClient, appSecurity, gitHubSettings, githubPaginatedHttpClient); + underTest = new GithubApplicationClientImpl(githubApplicationHttpClient, appSecurity, gitHubSettings, githubPaginatedHttpClient); logTester.clear(); } @@ -179,7 +179,7 @@ public class GithubApplicationClientImplTest { public void checkAppPermissions_IOException() throws IOException { AppToken appToken = mockAppToken(); - when(httpClient.get(appUrl, appToken, "/app")).thenThrow(new IOException("OOPS")); + when(githubApplicationHttpClient.get(appUrl, appToken, "/app")).thenThrow(new IOException("OOPS")); assertThatThrownBy(() -> underTest.checkAppPermissions(githubAppConfiguration)) .isInstanceOf(IllegalArgumentException.class) @@ -191,7 +191,7 @@ public class GithubApplicationClientImplTest { public void checkAppPermissions_ErrorCodes(int errorCode, String expectedMessage) throws IOException { AppToken appToken = mockAppToken(); - when(httpClient.get(appUrl, appToken, "/app")).thenReturn(new ErrorGetResponse(errorCode, null)); + when(githubApplicationHttpClient.get(appUrl, appToken, "/app")).thenReturn(new ErrorGetResponse(errorCode, null)); assertThatThrownBy(() -> underTest.checkAppPermissions(githubAppConfiguration)) .isInstanceOf(IllegalArgumentException.class) @@ -211,7 +211,7 @@ public class GithubApplicationClientImplTest { public void checkAppPermissions_MissingPermissions() throws IOException { AppToken appToken = mockAppToken(); - when(httpClient.get(appUrl, appToken, "/app")).thenReturn(new OkGetResponse("{}")); + when(githubApplicationHttpClient.get(appUrl, appToken, "/app")).thenReturn(new OkGetResponse("{}")); assertThatThrownBy(() -> underTest.checkAppPermissions(githubAppConfiguration)) .isInstanceOf(IllegalArgumentException.class) @@ -230,7 +230,7 @@ public class GithubApplicationClientImplTest { + " }\n" + "}"; - when(httpClient.get(appUrl, appToken, "/app")).thenReturn(new OkGetResponse(json)); + when(githubApplicationHttpClient.get(appUrl, appToken, "/app")).thenReturn(new OkGetResponse(json)); assertThatThrownBy(() -> underTest.checkAppPermissions(githubAppConfiguration)) .isInstanceOf(IllegalArgumentException.class) @@ -249,7 +249,7 @@ public class GithubApplicationClientImplTest { + " }\n" + "}"; - when(httpClient.get(appUrl, appToken, "/app")).thenReturn(new OkGetResponse(json)); + when(githubApplicationHttpClient.get(appUrl, appToken, "/app")).thenReturn(new OkGetResponse(json)); assertThatCode(() -> underTest.checkAppPermissions(githubAppConfiguration)).isNull(); } @@ -258,7 +258,7 @@ public class GithubApplicationClientImplTest { public void getInstallationId_returns_installation_id_of_given_account() throws IOException { AppToken appToken = new AppToken(APP_JWT_TOKEN); when(appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey())).thenReturn(appToken); - when(httpClient.get(appUrl, appToken, "/repos/torvalds/linux/installation")) + when(githubApplicationHttpClient.get(appUrl, appToken, "/repos/torvalds/linux/installation")) .thenReturn(new OkGetResponse("{" + " \"id\": 2," + " \"account\": {" + @@ -281,7 +281,7 @@ public class GithubApplicationClientImplTest { public void getInstallationId_return_empty_if_no_installation_found_for_githubAccount() throws IOException { AppToken appToken = new AppToken(APP_JWT_TOKEN); when(appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey())).thenReturn(appToken); - when(httpClient.get(appUrl, appToken, "/repos/torvalds/linux/installation")) + when(githubApplicationHttpClient.get(appUrl, appToken, "/repos/torvalds/linux/installation")) .thenReturn(new ErrorGetResponse(404, null)); assertThat(underTest.getInstallationId(githubAppConfiguration, "torvalds")).isEmpty(); @@ -290,44 +290,44 @@ public class GithubApplicationClientImplTest { @Test @UseDataProvider("githubServers") public void createUserAccessToken_returns_empty_if_access_token_cant_be_created(String apiUrl, String appUrl) throws IOException { - when(httpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) + when(githubApplicationHttpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) .thenReturn(new Response(400, null)); assertThatThrownBy(() -> underTest.createUserAccessToken(appUrl, "clientId", "clientSecret", "code")) .isInstanceOf(IllegalStateException.class); - verify(httpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); + verify(githubApplicationHttpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); } @Test @UseDataProvider("githubServers") public void createUserAccessToken_fail_if_access_token_request_fails(String apiUrl, String appUrl) throws IOException { - when(httpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) + when(githubApplicationHttpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) .thenThrow(new IOException("OOPS")); assertThatThrownBy(() -> underTest.createUserAccessToken(apiUrl, "clientId", "clientSecret", "code")) .isInstanceOf(IllegalStateException.class) .hasMessage("Failed to create GitHub's user access token"); - verify(httpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); + verify(githubApplicationHttpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); } @Test @UseDataProvider("githubServers") public void createUserAccessToken_throws_illegal_argument_exception_if_access_token_code_is_expired(String apiUrl, String appUrl) throws IOException { - when(httpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) + when(githubApplicationHttpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) .thenReturn(new OkGetResponse("error_code=100&error=expired_or_invalid")); assertThatThrownBy(() -> underTest.createUserAccessToken(apiUrl, "clientId", "clientSecret", "code")) .isInstanceOf(IllegalArgumentException.class); - verify(httpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); + verify(githubApplicationHttpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); } @Test @UseDataProvider("githubServers") public void createUserAccessToken_from_authorization_code_returns_access_token(String apiUrl, String appUrl) throws IOException { String token = randomAlphanumeric(10); - when(httpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) + when(githubApplicationHttpClient.post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code")) .thenReturn(new OkGetResponse("access_token=" + token + "&status=")); UserAccessToken userAccessToken = underTest.createUserAccessToken(apiUrl, "clientId", "clientSecret", "code"); @@ -335,14 +335,14 @@ public class GithubApplicationClientImplTest { assertThat(userAccessToken) .extracting(UserAccessToken::getValue, UserAccessToken::getAuthorizationHeaderPrefix) .containsOnly(token, "token"); - verify(httpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); + verify(githubApplicationHttpClient).post(appUrl, null, "/login/oauth/access_token?client_id=clientId&client_secret=clientSecret&code=code"); } @Test public void getApp_returns_id() throws IOException { AppToken appToken = new AppToken(APP_JWT_TOKEN); when(appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey())).thenReturn(appToken); - when(httpClient.get(appUrl, appToken, "/app")) + when(githubApplicationHttpClient.get(appUrl, appToken, "/app")) .thenReturn(new OkGetResponse("{\"installations_count\": 2}")); assertThat(underTest.getApp(githubAppConfiguration).getInstallationsCount()).isEqualTo(2L); @@ -352,7 +352,7 @@ public class GithubApplicationClientImplTest { public void getApp_whenStatusCodeIsNotOk_shouldThrowHttpException() throws IOException { AppToken appToken = new AppToken(APP_JWT_TOKEN); when(appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey())).thenReturn(appToken); - when(httpClient.get(appUrl, appToken, "/app")) + when(githubApplicationHttpClient.get(appUrl, appToken, "/app")) .thenReturn(new ErrorGetResponse(418, "I'm a teapot")); assertThatThrownBy(() -> underTest.getApp(githubAppConfiguration)) @@ -378,7 +378,7 @@ public class GithubApplicationClientImplTest { String appUrl = "https://github.sonarsource.com"; AccessToken accessToken = new UserAccessToken(randomAlphanumeric(10)); - when(httpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", 1, 100))) + when(githubApplicationHttpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", 1, 100))) .thenThrow(new IOException("OOPS")); assertThatThrownBy(() -> underTest.listOrganizations(appUrl, accessToken, 1, 100)) @@ -413,7 +413,7 @@ public class GithubApplicationClientImplTest { + " \"total_count\": 0\n" + "} "; - when(httpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", 1, 100))) + when(githubApplicationHttpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", 1, 100))) .thenReturn(new OkGetResponse(responseJson)); GithubApplicationClient.Organizations organizations = underTest.listOrganizations(appUrl, accessToken, 1, 100); @@ -504,7 +504,7 @@ public class GithubApplicationClientImplTest { + " ]\n" + "} "; - when(httpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", 1, 100))) + when(githubApplicationHttpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", 1, 100))) .thenReturn(new OkGetResponse(responseJson)); GithubApplicationClient.Organizations organizations = underTest.listOrganizations(appUrl, accessToken, 1, 100); @@ -581,18 +581,14 @@ public class GithubApplicationClientImplTest { } @Test - public void getWhitelistedGithubAppInstallations_whenGithubReturnsError_shouldThrow() throws IOException { + public void getWhitelistedGithubAppInstallations_whenGithubReturnsError_shouldReThrow() { AppToken appToken = new AppToken(APP_JWT_TOKEN); when(appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey())).thenReturn(appToken); - when(githubPaginatedHttpClient.get(any(), any(), any(), any())).thenThrow(new IOException("io exception")); + when(githubPaginatedHttpClient.get(any(), any(), any(), any())).thenThrow(new IllegalStateException("exception")); assertThatThrownBy(() -> underTest.getWhitelistedGithubAppInstallations(githubAppConfiguration)) .isInstanceOf(IllegalStateException.class) - .hasMessage( - "SonarQube was not able to retrieve resources from GitHub. " - + "This is likely due to a connectivity problem or a temporary network outage: " - + "Error while executing a paginated call to GitHub - appUrl: Any URL, path: /app/installations. io exception" - ); + .hasMessage("exception"); } @Test @@ -600,7 +596,7 @@ public class GithubApplicationClientImplTest { String appUrl = "https://github.sonarsource.com"; AccessToken accessToken = new UserAccessToken(randomAlphanumeric(10)); - when(httpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "org:test", 1, 100))) + when(githubApplicationHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "org:test", 1, 100))) .thenThrow(new IOException("OOPS")); assertThatThrownBy(() -> underTest.listRepositories(appUrl, accessToken, "test", null, 1, 100)) @@ -635,7 +631,7 @@ public class GithubApplicationClientImplTest { + " \"total_count\": 0\n" + "}"; - when(httpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "fork:true+org:github", 1, 100))) + when(githubApplicationHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "fork:true+org:github", 1, 100))) .thenReturn(new OkGetResponse(responseJson)); GithubApplicationClient.Repositories repositories = underTest.listRepositories(appUrl, accessToken, "github", null, 1, 100); @@ -723,7 +719,7 @@ public class GithubApplicationClientImplTest { + " ]\n" + "}"; - when(httpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "fork:true+org:github", 1, 100))) + when(githubApplicationHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "fork:true+org:github", 1, 100))) .thenReturn(new OkGetResponse(responseJson)); GithubApplicationClient.Repositories repositories = underTest.listRepositories(appUrl, accessToken, "github", null, 1, 100); @@ -778,7 +774,7 @@ public class GithubApplicationClientImplTest { + " ]\n" + "}"; - when(httpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "world+fork:true+org:github", 1, 100))) + when(githubApplicationHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", "world+fork:true+org:github", 1, 100))) .thenReturn(new GetResponse() { @Override public Optional getNextEndPoint() { @@ -811,7 +807,7 @@ public class GithubApplicationClientImplTest { @Test public void getRepository_returns_empty_when_repository_doesnt_exist() throws IOException { - when(httpClient.get(any(), any(), any())) + when(githubApplicationHttpClient.get(any(), any(), any())) .thenReturn(new Response(404, null)); Optional repository = underTest.getRepository(appUrl, new UserAccessToken("temp"), "octocat/Hello-World"); @@ -823,7 +819,7 @@ public class GithubApplicationClientImplTest { public void getRepository_fails_on_failure() throws IOException { String repositoryKey = "octocat/Hello-World"; - when(httpClient.get(any(), any(), any())) + when(githubApplicationHttpClient.get(any(), any(), any())) .thenThrow(new IOException("OOPS")); UserAccessToken token = new UserAccessToken("temp"); @@ -974,7 +970,7 @@ public class GithubApplicationClientImplTest { + " }" + "}"; - when(httpClient.get(appUrl, accessToken, "/repos/octocat/Hello-World")) + when(githubApplicationHttpClient.get(appUrl, accessToken, "/repos/octocat/Hello-World")) .thenReturn(new GetResponse() { @Override public Optional getNextEndPoint() { @@ -1022,7 +1018,7 @@ public class GithubApplicationClientImplTest { @Test public void createAppInstallationToken_returns_empty_if_post_throws_IOE() throws IOException { mockAppToken(); - when(httpClient.post(anyString(), any(AccessToken.class), anyString())).thenThrow(IOException.class); + when(githubApplicationHttpClient.post(anyString(), any(AccessToken.class), anyString())).thenThrow(IOException.class); Optional accessToken = underTest.createAppInstallationToken(githubAppConfiguration, INSTALLATION_ID); assertThat(accessToken).isEmpty(); @@ -1037,7 +1033,7 @@ public class GithubApplicationClientImplTest { Optional accessToken = underTest.createAppInstallationToken(githubAppConfiguration, INSTALLATION_ID); assertThat(accessToken).isEmpty(); - verify(httpClient).post(appUrl, appToken, "/app/installations/" + INSTALLATION_ID + "/access_tokens"); + verify(githubApplicationHttpClient).post(appUrl, appToken, "/app/installations/" + INSTALLATION_ID + "/access_tokens"); } @Test @@ -1048,7 +1044,7 @@ public class GithubApplicationClientImplTest { Optional accessToken = underTest.createAppInstallationToken(githubAppConfiguration, INSTALLATION_ID); assertThat(accessToken).hasValue(installToken); - verify(httpClient).post(appUrl, appToken, "/app/installations/" + INSTALLATION_ID + "/access_tokens"); + verify(githubApplicationHttpClient).post(appUrl, appToken, "/app/installations/" + INSTALLATION_ID + "/access_tokens"); } @Test @@ -1067,18 +1063,12 @@ public class GithubApplicationClientImplTest { } @Test - public void getRepositoryTeams_whenGitHubCallThrowsIOException_shouldLogAndThrow() throws IOException { - when(githubPaginatedHttpClient.get(eq(APP_URL), eq(appInstallationToken), eq(REPO_TEAMS_ENDPOINT), any())).thenThrow(new IOException("error")); + public void getRepositoryTeams_whenGitHubCallThrowsException_shouldRethrow() { + when(githubPaginatedHttpClient.get(eq(APP_URL), eq(appInstallationToken), eq(REPO_TEAMS_ENDPOINT), any())).thenThrow(new IllegalStateException("error")); assertThatIllegalStateException() .isThrownBy(() -> underTest.getRepositoryTeams(APP_URL, appInstallationToken, ORG_NAME, REPO_NAME)) - .isInstanceOf(IllegalStateException.class) - .withMessage( - "SonarQube was not able to retrieve resources from GitHub. This is likely due to a connectivity problem or a temporary network outage: Error while executing a paginated call to GitHub - appUrl: https://github.com/, path: /repos/ORG_NAME/repo1/teams. error"); - - assertThat(logTester.logs()).hasSize(1); - assertThat(logTester.logs(Level.WARN)) - .containsExactly("Error while executing a paginated call to GitHub - appUrl: https://github.com/, path: /repos/ORG_NAME/repo1/teams."); + .withMessage("error"); } private static List expectedTeams() { @@ -1104,19 +1094,12 @@ public class GithubApplicationClientImplTest { } @Test - public void getRepositoryCollaborators_whenGitHubCallThrowsIOException_shouldLogAndThrow() throws IOException { - when(githubPaginatedHttpClient.get(eq(APP_URL), eq(appInstallationToken), eq(REPO_COLLABORATORS_ENDPOINT), any())).thenThrow(new IOException("error")); + public void getRepositoryCollaborators_whenGitHubCallThrowsException_shouldRethrow() { + when(githubPaginatedHttpClient.get(eq(APP_URL), eq(appInstallationToken), eq(REPO_COLLABORATORS_ENDPOINT), any())).thenThrow(new IllegalStateException("error")); assertThatIllegalStateException() .isThrownBy(() -> underTest.getRepositoryCollaborators(APP_URL, appInstallationToken, ORG_NAME, REPO_NAME)) - .isInstanceOf(IllegalStateException.class) - .withMessage( - "SonarQube was not able to retrieve resources from GitHub. This is likely due to a connectivity problem or a temporary network outage: " - + "Error while executing a paginated call to GitHub - appUrl: https://github.com/, path: /repos/ORG_NAME/repo1/collaborators?affiliation=direct. error"); - - assertThat(logTester.logs()).hasSize(1); - assertThat(logTester.logs(Level.WARN)) - .containsExactly("Error while executing a paginated call to GitHub - appUrl: https://github.com/, path: /repos/ORG_NAME/repo1/collaborators?affiliation=direct."); + .withMessage("error"); } private static String getResponseContent(String path) throws IOException { @@ -1133,7 +1116,7 @@ public class GithubApplicationClientImplTest { Response response = mock(Response.class); when(response.getContent()).thenReturn(Optional.empty()); when(response.getCode()).thenReturn(HTTP_UNAUTHORIZED); - when(httpClient.post(eq(appUrl), any(AppToken.class), eq("/app/installations/" + INSTALLATION_ID + "/access_tokens"))).thenReturn(response); + when(githubApplicationHttpClient.post(eq(appUrl), any(AppToken.class), eq("/app/installations/" + INSTALLATION_ID + "/access_tokens"))).thenReturn(response); } private AppToken mockAppToken() { @@ -1149,7 +1132,7 @@ public class GithubApplicationClientImplTest { " \"token\": \"" + token + "\"" + "}")); when(response.getCode()).thenReturn(HTTP_CREATED); - when(httpClient.post(eq(appUrl), any(AppToken.class), eq("/app/installations/" + INSTALLATION_ID + "/access_tokens"))).thenReturn(response); + when(githubApplicationHttpClient.post(eq(appUrl), any(AppToken.class), eq("/app/installations/" + INSTALLATION_ID + "/access_tokens"))).thenReturn(response); return new AppInstallationToken(token); } diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabHttpClientTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabApplicationClientTest.java similarity index 88% rename from server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabHttpClientTest.java rename to server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabApplicationClientTest.java index 48c0b2d0067..4ec9f5d33fe 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabHttpClientTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabApplicationClientTest.java @@ -20,31 +20,46 @@ package org.sonar.alm.client.gitlab; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; +import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.slf4j.event.Level; import org.sonar.alm.client.ConstantTimeoutConfiguration; import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.auth.gitlab.GsonGroup; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; -public class GitlabHttpClientTest { +public class GitlabApplicationClientTest { @Rule public LogTester logTester = new LogTester(); + + private GitlabPaginatedHttpClient gitlabPaginatedHttpClient = mock(); + private final MockWebServer server = new MockWebServer(); - private GitlabHttpClient underTest; + private GitlabApplicationClient underTest; private String gitlabUrl; @Before @@ -54,7 +69,7 @@ public class GitlabHttpClientTest { gitlabUrl = urlWithEndingSlash.substring(0, urlWithEndingSlash.length() - 1); TimeoutConfiguration timeoutConfiguration = new ConstantTimeoutConfiguration(10_000); - underTest = new GitlabHttpClient(timeoutConfiguration); + underTest = new GitlabApplicationClient(gitlabPaginatedHttpClient, timeoutConfiguration); } @After @@ -523,4 +538,52 @@ public class GitlabHttpClientTest { + "] " + "failed with error message : [Failed to connect to " + server.getHostName()); } + + @Test + public void getGroups_whenCallIsInError_rethrows() throws IOException { + String token = "token-toto"; + GitlabToken gitlabToken = new GitlabToken(token); + when(gitlabPaginatedHttpClient.get(eq(gitlabUrl), eq(gitlabToken), eq("/groups"), any())).thenThrow(new IllegalStateException("exception")); + + assertThatIllegalStateException() + .isThrownBy(() -> underTest.getGroups(gitlabUrl, token)) + .withMessage("exception"); + } + + @Test + public void getGroups_whenCallIsSuccessful_deserializesAndReturnsCorrectlyGroups() throws IOException { + ArgumentCaptor>> deserializerCaptor = ArgumentCaptor.forClass(Function.class); + + String token = "token-toto"; + GitlabToken gitlabToken = new GitlabToken(token); + List expectedGroups = expectedGroups(); + when(gitlabPaginatedHttpClient.get(eq(gitlabUrl), eq(gitlabToken), eq("/groups"), deserializerCaptor.capture())).thenReturn(expectedGroups); + + Set groups = underTest.getGroups(gitlabUrl, token); + assertThat(groups).containsExactlyInAnyOrderElementsOf(expectedGroups); + + String responseContent = getResponseContent("groups-full-response.json"); + + List deserializedGroups = deserializerCaptor.getValue().apply(responseContent); + assertThat(deserializedGroups).usingRecursiveComparison().isEqualTo(expectedGroups); + } + + private static List expectedGroups() { + GsonGroup gsonGroup = createGsonGroup("56232243", "sonarsource/cfamily", "this is a long description"); + GsonGroup gsonGroup2 = createGsonGroup("78902256", "sonarsource/sonarqube/mmf-3052-ant1", ""); + return List.of(gsonGroup, gsonGroup2); + } + + private static GsonGroup createGsonGroup(String number, String fullPath, String description) { + GsonGroup gsonGroup = mock(GsonGroup.class); + when(gsonGroup.getId()).thenReturn(number); + when(gsonGroup.getFullPath()).thenReturn(fullPath); + when(gsonGroup.getDescription()).thenReturn(description); + return gsonGroup; + } + + private static String getResponseContent(String path) throws IOException { + return IOUtils.toString(GitlabApplicationClientTest.class.getResourceAsStream(path), StandardCharsets.UTF_8); + } + } diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidatorTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidatorTest.java index 6582f503024..13f1c4c9c47 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidatorTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidatorTest.java @@ -21,8 +21,6 @@ package org.sonar.alm.client.gitlab; import org.junit.BeforeClass; import org.junit.Test; -import org.sonar.alm.client.gitlab.GitlabGlobalSettingsValidator; -import org.sonar.alm.client.gitlab.GitlabHttpClient; import org.sonar.api.config.internal.Encryption; import org.sonar.api.config.internal.Settings; import org.sonar.db.alm.setting.AlmSettingDto; @@ -37,7 +35,7 @@ public class GitlabGlobalSettingsValidatorTest { private static final Encryption encryption = mock(Encryption.class); private static final Settings settings = mock(Settings.class); - private final GitlabHttpClient gitlabHttpClient = mock(GitlabHttpClient.class); + private final GitlabApplicationClient gitlabHttpClient = mock(GitlabApplicationClient.class); private final GitlabGlobalSettingsValidator underTest = new GitlabGlobalSettingsValidator(gitlabHttpClient, settings); diff --git a/server/sonar-alm-client/src/test/resources/org/sonar/alm/client/gitlab/groups-full-response.json b/server/sonar-alm-client/src/test/resources/org/sonar/alm/client/gitlab/groups-full-response.json new file mode 100644 index 00000000000..45861a200e6 --- /dev/null +++ b/server/sonar-alm-client/src/test/resources/org/sonar/alm/client/gitlab/groups-full-response.json @@ -0,0 +1,76 @@ +[ + { + "id": 56232243, + "web_url": "https://gitlab.com/groups/sonarsource/cfamily", + "name": "CFamily", + "path": "cfamily", + "description": "this is a long description", + "visibility": "public", + "share_with_group_lock": false, + "require_two_factor_authentication": false, + "two_factor_grace_period": 48, + "project_creation_level": "maintainer", + "auto_devops_enabled": null, + "subgroup_creation_level": "owner", + "emails_disabled": false, + "emails_enabled": true, + "mentions_disabled": null, + "lfs_enabled": false, + "default_branch_protection": 2, + "default_branch_protection_defaults": { + "allowed_to_push": [ + { + "access_level": 30 + } + ], + "allow_force_push": true, + "allowed_to_merge": [ + { + "access_level": 30 + } + ] + }, + "avatar_url": null, + "request_access_enabled": false, + "full_name": "SonarSource / CFamily", + "full_path": "sonarsource/cfamily", + "created_at": "2022-08-02T06:56:14.451Z", + "parent_id": 6164984, + "shared_runners_setting": "enabled", + "ldap_cn": null, + "ldap_access": null, + "marked_for_deletion_on": null, + "wiki_access_level": "enabled" + }, + { + "id": 78902256, + "web_url": "https://gitlab.com/groups/sonarsource/sonarqube/mmf-3052-ant1", + "name": "MMF-3052-Ant1", + "path": "mmf-3052-ant1", + "description": "", + "visibility": "private", + "share_with_group_lock": true, + "require_two_factor_authentication": false, + "two_factor_grace_period": 48, + "project_creation_level": "developer", + "auto_devops_enabled": null, + "subgroup_creation_level": "maintainer", + "emails_disabled": false, + "emails_enabled": true, + "mentions_disabled": null, + "lfs_enabled": true, + "default_branch_protection": 2, + "default_branch_protection_defaults": {}, + "avatar_url": null, + "request_access_enabled": true, + "full_name": "SonarSource / SonarQube / MMF-3052-Ant1", + "full_path": "sonarsource/sonarqube/mmf-3052-ant1", + "created_at": "2023-11-29T10:34:43.382Z", + "parent_id": 67918039, + "shared_runners_setting": "enabled", + "ldap_cn": null, + "ldap_access": null, + "marked_for_deletion_on": null, + "wiki_access_level": "enabled" + } +] diff --git a/server/sonar-auth-gitlab/src/main/java/org/sonar/auth/gitlab/GsonGroup.java b/server/sonar-auth-gitlab/src/main/java/org/sonar/auth/gitlab/GsonGroup.java index b4ebf1a666d..2fe325c2ab4 100644 --- a/server/sonar-auth-gitlab/src/main/java/org/sonar/auth/gitlab/GsonGroup.java +++ b/server/sonar-auth-gitlab/src/main/java/org/sonar/auth/gitlab/GsonGroup.java @@ -31,22 +31,36 @@ import java.util.List; */ public class GsonGroup { + @SerializedName("id") + private String id; @SerializedName("full_path") private String fullPath; + @SerializedName("description") + private String description; public GsonGroup() { // http://stackoverflow.com/a/18645370/229031 - this(""); + this("", "", ""); } - GsonGroup(String fullPath) { + private GsonGroup(String id, String fullPath, String description) { + this.id = id; this.fullPath = fullPath; + this.description = description; } - String getFullPath() { + public String getId() { + return id; + } + + public String getFullPath() { return fullPath; } + public String getDescription() { + return description; + } + static List parse(String json) { Type collectionType = new TypeToken>() { }.getType(); diff --git a/server/sonar-auth-gitlab/src/test/java/org/sonar/auth/gitlab/GsonGroupTest.java b/server/sonar-auth-gitlab/src/test/java/org/sonar/auth/gitlab/GsonGroupTest.java index 384fad4d84b..b72f834c16e 100644 --- a/server/sonar-auth-gitlab/src/test/java/org/sonar/auth/gitlab/GsonGroupTest.java +++ b/server/sonar-auth-gitlab/src/test/java/org/sonar/auth/gitlab/GsonGroupTest.java @@ -33,7 +33,7 @@ public class GsonGroupTest { "\"web_url\": \"https://gitlab.com/groups/my-awesome-group/my-project\",\n" + "\"name\": \"my-project\",\n" + "\"path\": \"my-project\",\n" + - "\"description\": \"\",\n" + + "\"description\": \"toto\",\n" + "\"visibility\": \"private\",\n" + "\"lfs_enabled\": true,\n" + "\"avatar_url\": null,\n" + @@ -47,6 +47,8 @@ public class GsonGroupTest { assertThat(groups).isNotNull(); assertThat(groups.size()).isOne(); + assertThat(groups.get(0).getId()).isEqualTo("123456789"); assertThat(groups.get(0).getFullPath()).isEqualTo("my-awesome-group/my-project"); + assertThat(groups.get(0).getDescription()).isEqualTo("toto"); } } diff --git a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/CheckPatActionIT.java b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/CheckPatActionIT.java index 7bac4b067ce..dc553ca9a5a 100644 --- a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/CheckPatActionIT.java +++ b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/CheckPatActionIT.java @@ -24,7 +24,7 @@ import org.junit.Test; import org.sonar.alm.client.azure.AzureDevOpsHttpClient; import org.sonar.alm.client.bitbucket.bitbucketcloud.BitbucketCloudRestClient; import org.sonar.alm.client.bitbucketserver.BitbucketServerRestClient; -import org.sonar.alm.client.gitlab.GitlabHttpClient; +import org.sonar.alm.client.gitlab.GitlabApplicationClient; import org.sonar.api.server.ws.WebService; import org.sonar.db.DbTester; import org.sonar.db.alm.pat.AlmPatDto; @@ -60,9 +60,9 @@ public class CheckPatActionIT { private final AzureDevOpsHttpClient azureDevOpsPrHttpClient = mock(AzureDevOpsHttpClient.class); private final BitbucketCloudRestClient bitbucketCloudRestClient = mock(BitbucketCloudRestClient.class); private final BitbucketServerRestClient bitbucketServerRestClient = mock(BitbucketServerRestClient.class); - private final GitlabHttpClient gitlabPrHttpClient = mock(GitlabHttpClient.class); + private final GitlabApplicationClient gitlabApplicationClient = mock(GitlabApplicationClient.class); private final WsActionTester ws = new WsActionTester(new CheckPatAction(db.getDbClient(), userSession, azureDevOpsPrHttpClient, - bitbucketCloudRestClient, bitbucketServerRestClient, gitlabPrHttpClient)); + bitbucketCloudRestClient, bitbucketServerRestClient, gitlabApplicationClient)); @Test public void check_pat_for_github() { @@ -134,7 +134,7 @@ public class CheckPatActionIT { .execute(); assertThat(almSetting.getUrl()).isNotNull(); - verify(gitlabPrHttpClient).searchProjects(almSetting.getUrl(), PAT_SECRET, null, null, null); + verify(gitlabApplicationClient).searchProjects(almSetting.getUrl(), PAT_SECRET, null, null, null); } @Test @@ -175,7 +175,7 @@ public class CheckPatActionIT { @Test public void fail_when_personal_access_token_is_invalid_for_gitlab() { - when(gitlabPrHttpClient.searchProjects(any(), any(), any(), any(), any())) + when(gitlabApplicationClient.searchProjects(any(), any(), any(), any(), any())) .thenThrow(new IllegalArgumentException("Invalid personal access token")); UserDto user = db.users().insertUser(); userSession.logIn(user).addPermission(PROVISION_PROJECTS); diff --git a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectActionIT.java b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectActionIT.java index b1c8414e7db..c38535314cd 100644 --- a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectActionIT.java +++ b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectActionIT.java @@ -26,7 +26,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.sonar.alm.client.gitlab.GitLabBranch; -import org.sonar.alm.client.gitlab.GitlabHttpClient; +import org.sonar.alm.client.gitlab.GitlabApplicationClient; import org.sonar.alm.client.gitlab.Project; import org.sonar.api.utils.System2; import org.sonar.core.i18n.I18n; @@ -94,14 +94,14 @@ public class ImportGitLabProjectActionIT { mock(PermissionTemplateService.class), new FavoriteUpdater(db.getDbClient()), new TestIndexers(), new SequenceUuidFactory(), defaultBranchNameResolver, mock(PermissionUpdater.class), mock(PermissionService.class)); - private final GitlabHttpClient gitlabHttpClient = mock(GitlabHttpClient.class); + private final GitlabApplicationClient gitlabApplicationClient = mock(GitlabApplicationClient.class); private final ImportHelper importHelper = new ImportHelper(db.getDbClient(), userSession); private final ProjectDefaultVisibility projectDefaultVisibility = mock(ProjectDefaultVisibility.class); private final ProjectKeyGenerator projectKeyGenerator = mock(ProjectKeyGenerator.class); private PlatformEditionProvider editionProvider = mock(PlatformEditionProvider.class); private NewCodeDefinitionResolver newCodeDefinitionResolver = new NewCodeDefinitionResolver(db.getDbClient(), editionProvider); private final ImportGitLabProjectAction importGitLabProjectAction = new ImportGitLabProjectAction( - db.getDbClient(), userSession, projectDefaultVisibility, gitlabHttpClient, componentUpdater, importHelper, projectKeyGenerator, newCodeDefinitionResolver, + db.getDbClient(), userSession, projectDefaultVisibility, gitlabApplicationClient, componentUpdater, importHelper, projectKeyGenerator, newCodeDefinitionResolver, defaultBranchNameResolver); private final WsActionTester ws = new WsActionTester(importGitLabProjectAction); @@ -125,7 +125,7 @@ public class ImportGitLabProjectActionIT { .setParam(PARAM_NEW_CODE_DEFINITION_VALUE, "30") .executeProtobuf(Projects.CreateWsResponse.class); - verify(gitlabHttpClient).getProject(almSetting.getUrl(), "PAT", 12345L); + verify(gitlabApplicationClient).getProject(almSetting.getUrl(), "PAT", 12345L); Projects.CreateWsResponse.Project result = response.getProject(); assertThat(result.getKey()).isEqualTo(PROJECT_KEY_NAME); @@ -179,8 +179,8 @@ public class ImportGitLabProjectActionIT { .setParam("gitlabProjectId", "12345") .executeProtobuf(Projects.CreateWsResponse.class); - verify(gitlabHttpClient).getProject(almSetting.getUrl(), "PAT", 12345L); - verify(gitlabHttpClient).getBranches(almSetting.getUrl(), "PAT", 12345L); + verify(gitlabApplicationClient).getProject(almSetting.getUrl(), "PAT", 12345L); + verify(gitlabApplicationClient).getBranches(almSetting.getUrl(), "PAT", 12345L); Projects.CreateWsResponse.Project result = response.getProject(); assertThat(result.getKey()).isEqualTo(PROJECT_KEY_NAME); @@ -205,8 +205,8 @@ public class ImportGitLabProjectActionIT { .setParam("gitlabProjectId", "12345") .executeProtobuf(Projects.CreateWsResponse.class); - verify(gitlabHttpClient).getProject(almSetting.getUrl(), "PAT", 12345L); - verify(gitlabHttpClient).getBranches(almSetting.getUrl(), "PAT", 12345L); + verify(gitlabApplicationClient).getProject(almSetting.getUrl(), "PAT", 12345L); + verify(gitlabApplicationClient).getBranches(almSetting.getUrl(), "PAT", 12345L); Projects.CreateWsResponse.Project result = response.getProject(); assertThat(result.getKey()).isEqualTo(PROJECT_KEY_NAME); @@ -231,7 +231,7 @@ public class ImportGitLabProjectActionIT { .setParam("gitlabProjectId", "12345") .executeProtobuf(Projects.CreateWsResponse.class); - verify(gitlabHttpClient).getProject(almSetting.getUrl(), "PAT", 12345L); + verify(gitlabApplicationClient).getProject(almSetting.getUrl(), "PAT", 12345L); Projects.CreateWsResponse.Project result = response.getProject(); assertThat(result.getKey()).isEqualTo(PROJECT_KEY_NAME); @@ -344,8 +344,8 @@ public class ImportGitLabProjectActionIT { private Project mockGitlabProject(List master) { Project project = new Project(randomAlphanumeric(5), randomAlphanumeric(5)); - when(gitlabHttpClient.getProject(any(), any(), any())).thenReturn(project); - when(gitlabHttpClient.getBranches(any(), any(), any())).thenReturn(master); + when(gitlabApplicationClient.getProject(any(), any(), any())).thenReturn(project); + when(gitlabApplicationClient.getBranches(any(), any(), any())).thenReturn(master); when(projectKeyGenerator.generateUniqueProjectKey(project.getPathWithNamespace())).thenReturn(PROJECT_KEY_NAME); return project; } diff --git a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposActionIT.java b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposActionIT.java index 95707c614b6..219d4208bbd 100644 --- a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposActionIT.java +++ b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposActionIT.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.LinkedList; import org.junit.Rule; import org.junit.Test; -import org.sonar.alm.client.gitlab.GitlabHttpClient; +import org.sonar.alm.client.gitlab.GitlabApplicationClient; import org.sonar.alm.client.gitlab.Project; import org.sonar.alm.client.gitlab.ProjectList; import org.sonar.api.server.ws.WebService; @@ -59,9 +59,9 @@ public class SearchGitlabReposActionIT { @Rule public DbTester db = DbTester.create(); - private final GitlabHttpClient gitlabHttpClient = mock(GitlabHttpClient.class); + private final GitlabApplicationClient gitlabApplicationClient = mock(GitlabApplicationClient.class); private final WsActionTester ws = new WsActionTester(new SearchGitlabReposAction(db.getDbClient(), userSession, - gitlabHttpClient)); + gitlabApplicationClient)); @Test public void list_gitlab_repos() { @@ -69,7 +69,7 @@ public class SearchGitlabReposActionIT { Project gitlabProject2 = new Project(2L, "repoName2", "path1 / repoName2", "repo-slug-2", "path-1/repo-slug-2", "url-2"); Project gitlabProject3 = new Project(3L, "repoName3", "repoName3 / repoName3", "repo-slug-3", "repo-slug-3/repo-slug-3", "url-3"); Project gitlabProject4 = new Project(4L, "repoName4", "repoName4 / repoName4 / repoName4", "repo-slug-4", "repo-slug-4/repo-slug-4/repo-slug-4", "url-4"); - when(gitlabHttpClient.searchProjects(any(), any(), any(), anyInt(), anyInt())) + when(gitlabApplicationClient.searchProjects(any(), any(), any(), anyInt(), anyInt())) .thenReturn( new ProjectList(Arrays.asList(gitlabProject1, gitlabProject2, gitlabProject3, gitlabProject4), 1, 10, 4)); @@ -112,7 +112,7 @@ public class SearchGitlabReposActionIT { Project gitlabProject2 = new Project(2L, "repoName2", "path1 / repoName2", "repo-slug-2", "path-1/repo-slug-2", "url-2"); Project gitlabProject3 = new Project(3L, "repoName3", "repoName3 / repoName3", "repo-slug-3", "repo-slug-3/repo-slug-3", "url-3"); Project gitlabProject4 = new Project(4L, "repoName4", "repoName4 / repoName4 / repoName4", "repo-slug-4", "repo-slug-4/repo-slug-4/repo-slug-4", "url-4"); - when(gitlabHttpClient.searchProjects(any(), any(), any(), anyInt(), anyInt())) + when(gitlabApplicationClient.searchProjects(any(), any(), any(), anyInt(), anyInt())) .thenReturn( new ProjectList(Arrays.asList(gitlabProject1, gitlabProject2, gitlabProject3, gitlabProject4), 1, 10, 4)); @@ -160,7 +160,7 @@ public class SearchGitlabReposActionIT { @Test public void return_empty_list_when_no_gitlab_projects() { - when(gitlabHttpClient.searchProjects(any(), any(), any(), anyInt(), anyInt())).thenReturn(new ProjectList(new LinkedList<>(), 1, 10, 0)); + when(gitlabApplicationClient.searchProjects(any(), any(), any(), anyInt(), anyInt())).thenReturn(new ProjectList(new LinkedList<>(), 1, 10, 0)); UserDto user = db.users().insertUser(); userSession.logIn(user).addPermission(PROVISION_PROJECTS); AlmSettingDto almSetting = db.almSettings().insertBitbucketAlmSetting(); @@ -238,7 +238,7 @@ public class SearchGitlabReposActionIT { "https://example.gitlab.com/group/gitlab-repo-name-2"); Project gitlabProject3 = new Project(3L, "Gitlab repo name 3", "Group / Gitlab repo name 3", "gitlab-repo-name-3", "group/gitlab-repo-name-3", "https://example.gitlab.com/group/gitlab-repo-name-3"); - when(gitlabHttpClient.searchProjects(any(), any(), any(), anyInt(), anyInt())) + when(gitlabApplicationClient.searchProjects(any(), any(), any(), anyInt(), anyInt())) .thenReturn( new ProjectList(Arrays.asList(gitlabProject1, gitlabProject2, gitlabProject3), 1, 3, 10)); diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/CheckPatAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/CheckPatAction.java index a9fc2ad8d6d..212bf09c2c7 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/CheckPatAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/CheckPatAction.java @@ -22,7 +22,7 @@ package org.sonar.server.almintegration.ws; import org.sonar.alm.client.azure.AzureDevOpsHttpClient; import org.sonar.alm.client.bitbucket.bitbucketcloud.BitbucketCloudRestClient; import org.sonar.alm.client.bitbucketserver.BitbucketServerRestClient; -import org.sonar.alm.client.gitlab.GitlabHttpClient; +import org.sonar.alm.client.gitlab.GitlabApplicationClient; import org.sonar.api.server.ws.Change; import org.sonar.api.server.ws.Request; import org.sonar.api.server.ws.Response; @@ -50,19 +50,19 @@ public class CheckPatAction implements AlmIntegrationsWsAction { private final AzureDevOpsHttpClient azureDevOpsHttpClient; private final BitbucketCloudRestClient bitbucketCloudRestClient; private final BitbucketServerRestClient bitbucketServerRestClient; - private final GitlabHttpClient gitlabHttpClient; + private final GitlabApplicationClient gitlabApplicationClient; public CheckPatAction(DbClient dbClient, UserSession userSession, AzureDevOpsHttpClient azureDevOpsHttpClient, BitbucketCloudRestClient bitbucketCloudRestClient, BitbucketServerRestClient bitbucketServerRestClient, - GitlabHttpClient gitlabHttpClient) { + GitlabApplicationClient gitlabApplicationClient) { this.dbClient = dbClient; this.userSession = userSession; this.azureDevOpsHttpClient = azureDevOpsHttpClient; this.bitbucketCloudRestClient = bitbucketCloudRestClient; this.bitbucketServerRestClient = bitbucketServerRestClient; - this.gitlabHttpClient = gitlabHttpClient; + this.gitlabApplicationClient = gitlabApplicationClient; } @Override @@ -113,7 +113,7 @@ public class CheckPatAction implements AlmIntegrationsWsAction { requireNonNull(almPatDto.getPersonalAccessToken(), PAT_CANNOT_BE_NULL)); break; case GITLAB: - gitlabHttpClient.searchProjects( + gitlabApplicationClient.searchProjects( requireNonNull(almSettingDto.getUrl(), URL_CANNOT_BE_NULL), requireNonNull(almPatDto.getPersonalAccessToken(), PAT_CANNOT_BE_NULL), null, null, null); diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectAction.java index c532245f5e3..e5e0318427c 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/ImportGitLabProjectAction.java @@ -23,7 +23,7 @@ import java.util.Optional; import javax.annotation.Nullable; import javax.inject.Inject; import org.sonar.alm.client.gitlab.GitLabBranch; -import org.sonar.alm.client.gitlab.GitlabHttpClient; +import org.sonar.alm.client.gitlab.GitlabApplicationClient; import org.sonar.alm.client.gitlab.Project; import org.sonar.api.server.ws.Change; import org.sonar.api.server.ws.Request; @@ -70,7 +70,7 @@ public class ImportGitLabProjectAction implements AlmIntegrationsWsAction { private final DbClient dbClient; private final UserSession userSession; private final ProjectDefaultVisibility projectDefaultVisibility; - private final GitlabHttpClient gitlabHttpClient; + private final GitlabApplicationClient gitlabApplicationClient; private final ComponentUpdater componentUpdater; private final ImportHelper importHelper; private final ProjectKeyGenerator projectKeyGenerator; @@ -79,13 +79,13 @@ public class ImportGitLabProjectAction implements AlmIntegrationsWsAction { @Inject public ImportGitLabProjectAction(DbClient dbClient, UserSession userSession, - ProjectDefaultVisibility projectDefaultVisibility, GitlabHttpClient gitlabHttpClient, + ProjectDefaultVisibility projectDefaultVisibility, GitlabApplicationClient gitlabApplicationClient, ComponentUpdater componentUpdater, ImportHelper importHelper, ProjectKeyGenerator projectKeyGenerator, NewCodeDefinitionResolver newCodeDefinitionResolver, DefaultBranchNameResolver defaultBranchNameResolver) { this.dbClient = dbClient; this.userSession = userSession; this.projectDefaultVisibility = projectDefaultVisibility; - this.gitlabHttpClient = gitlabHttpClient; + this.gitlabApplicationClient = gitlabApplicationClient; this.componentUpdater = componentUpdater; this.importHelper = importHelper; this.projectKeyGenerator = projectKeyGenerator; @@ -139,7 +139,7 @@ public class ImportGitLabProjectAction implements AlmIntegrationsWsAction { long gitlabProjectId = request.mandatoryParamAsLong(PARAM_GITLAB_PROJECT_ID); String gitlabUrl = requireNonNull(almSettingDto.getUrl(), "DevOps Platform gitlabUrl cannot be null"); - Project gitlabProject = gitlabHttpClient.getProject(gitlabUrl, pat, gitlabProjectId); + Project gitlabProject = gitlabApplicationClient.getProject(gitlabUrl, pat, gitlabProjectId); Optional almMainBranchName = getAlmDefaultBranch(pat, gitlabProjectId, gitlabUrl); @@ -169,7 +169,7 @@ public class ImportGitLabProjectAction implements AlmIntegrationsWsAction { } private Optional getAlmDefaultBranch(String pat, long gitlabProjectId, String gitlabUrl) { - Optional almMainBranch = gitlabHttpClient.getBranches(gitlabUrl, pat, gitlabProjectId).stream().filter(GitLabBranch::isDefault).findFirst(); + Optional almMainBranch = gitlabApplicationClient.getBranches(gitlabUrl, pat, gitlabProjectId).stream().filter(GitLabBranch::isDefault).findFirst(); return almMainBranch.map(GitLabBranch::getName); } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposAction.java index a27f3280dc4..223421dde73 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/almintegration/ws/gitlab/SearchGitlabReposAction.java @@ -26,7 +26,7 @@ import java.util.Set; import java.util.function.BinaryOperator; import java.util.function.Function; import java.util.stream.Collectors; -import org.sonar.alm.client.gitlab.GitlabHttpClient; +import org.sonar.alm.client.gitlab.GitlabApplicationClient; import org.sonar.alm.client.gitlab.Project; import org.sonar.alm.client.gitlab.ProjectList; import org.sonar.api.server.ws.Request; @@ -58,12 +58,12 @@ public class SearchGitlabReposAction implements AlmIntegrationsWsAction { private final DbClient dbClient; private final UserSession userSession; - private final GitlabHttpClient gitlabHttpClient; + private final GitlabApplicationClient gitlabApplicationClient; - public SearchGitlabReposAction(DbClient dbClient, UserSession userSession, GitlabHttpClient gitlabHttpClient) { + public SearchGitlabReposAction(DbClient dbClient, UserSession userSession, GitlabApplicationClient gitlabApplicationClient) { this.dbClient = dbClient; this.userSession = userSession; - this.gitlabHttpClient = gitlabHttpClient; + this.gitlabApplicationClient = gitlabApplicationClient; } @Override @@ -113,7 +113,7 @@ public class SearchGitlabReposAction implements AlmIntegrationsWsAction { String personalAccessToken = almPatDto.map(AlmPatDto::getPersonalAccessToken).orElseThrow(() -> new IllegalArgumentException("No personal access token found")); String gitlabUrl = requireNonNull(almSettingDto.getUrl(), "DevOps Platform url cannot be null"); - ProjectList gitlabProjectList = gitlabHttpClient + ProjectList gitlabProjectList = gitlabApplicationClient .searchProjects(gitlabUrl, personalAccessToken, projectName, pageNumber, pageSize); Map sqProjectsKeyByGitlabProjectId = getSqProjectsKeyByGitlabProjectId(dbSession, almSettingDto, gitlabProjectList); diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index fc9b920c860..a57327580b2 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -33,11 +33,14 @@ import org.sonar.alm.client.github.GithubGlobalSettingsValidator; import org.sonar.alm.client.github.GithubHeaders; import org.sonar.alm.client.github.GithubPaginatedHttpClient; import org.sonar.alm.client.github.GithubPermissionConverter; -import org.sonar.alm.client.github.RatioBasedRateLimitChecker; +import org.sonar.alm.client.RatioBasedRateLimitChecker; import org.sonar.alm.client.github.config.GithubProvisioningConfigValidator; import org.sonar.alm.client.github.security.GithubAppSecurityImpl; +import org.sonar.alm.client.gitlab.GitlabApplicationHttpClient; import org.sonar.alm.client.gitlab.GitlabGlobalSettingsValidator; -import org.sonar.alm.client.gitlab.GitlabHttpClient; +import org.sonar.alm.client.gitlab.GitlabHeaders; +import org.sonar.alm.client.gitlab.GitlabApplicationClient; +import org.sonar.alm.client.gitlab.GitlabPaginatedHttpClient; import org.sonar.api.resources.ResourceTypes; import org.sonar.api.server.rule.RulesDefinitionXmlLoader; import org.sonar.auth.bitbucket.BitbucketModule; @@ -555,22 +558,25 @@ public class PlatformLevel4 extends PlatformLevel { ProjectKeyGenerator.class, RatioBasedRateLimitChecker.class, GithubAppSecurityImpl.class, - GithubApplicationClientImpl.class, - GithubPaginatedHttpClient.class, GithubHeaders.class, GithubApplicationHttpClient.class, + GithubPaginatedHttpClient.class, + GithubApplicationClientImpl.class, GithubProvisioningConfigValidator.class, GithubProvisioningWs.class, GithubProjectCreatorFactory.class, GithubPermissionConverter.class, BitbucketCloudRestClientConfiguration.class, BitbucketServerRestClient.class, - GitlabHttpClient.class, AzureDevOpsHttpClient.class, new AlmIntegrationsWSModule(), BitbucketCloudValidator.class, BitbucketServerSettingsValidator.class, GithubGlobalSettingsValidator.class, + GitlabHeaders.class, + GitlabApplicationHttpClient.class, + GitlabPaginatedHttpClient.class, + GitlabApplicationClient.class, GitlabGlobalSettingsValidator.class, AzureDevOpsValidator.class, -- 2.39.5