diff options
author | Aurelien Poscia <aurelien.poscia@sonarsource.com> | 2023-11-30 09:58:20 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2023-12-22 20:03:01 +0000 |
commit | 68de595dc688e9fd09bd8925c94d0bba830c3869 (patch) | |
tree | 6ca9621267a13fcc2a752dc2a882b60057ee7761 /server/sonar-alm-client/src/main/java | |
parent | c0c9226eb421d0581b269c74a083aad00a7ad679 (diff) | |
download | sonarqube-68de595dc688e9fd09bd8925c94d0bba830c3869.tar.gz sonarqube-68de595dc688e9fd09bd8925c94d0bba830c3869.zip |
SONAR-21119 Provide method to get groups for GitLab & refactored GithubPaginatedHttpClient and GithubApplicationHttpClient to make them generic
Diffstat (limited to 'server/sonar-alm-client/src/main/java')
16 files changed, 355 insertions, 152 deletions
diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/ApplicationHttpClient.java index 75c512eddb5..933af0910f1 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/ApplicationHttpClient.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import java.io.IOException; import java.util.Optional; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/DevopsPlatformHeaders.java index 38e8fe0b94c..210a562b260 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/DevopsPlatformHeaders.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import java.util.Optional; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericApplicationHttpClient.java index d67224c2615..fc375f96992 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericApplicationHttpClient.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import java.io.IOException; import java.net.MalformedURLException; @@ -37,7 +37,6 @@ import okhttp3.ResponseBody; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.alm.client.github.security.AccessToken; import org.sonarqube.ws.client.OkHttpClientBuilder; @@ -58,7 +57,7 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli private final DevopsPlatformHeaders devopsPlatformHeaders; private final OkHttpClient client; - public GenericApplicationHttpClient(DevopsPlatformHeaders devopsPlatformHeaders, TimeoutConfiguration timeoutConfiguration) { + protected GenericApplicationHttpClient(DevopsPlatformHeaders devopsPlatformHeaders, TimeoutConfiguration timeoutConfiguration) { this.devopsPlatformHeaders = devopsPlatformHeaders; client = new OkHttpClientBuilder() .setConnectTimeoutMs(timeoutConfiguration.getConnectTimeout()) @@ -184,7 +183,7 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli private Request.Builder newRequestBuilder(String appUrl, @Nullable AccessToken token, String endPoint) { Request.Builder url = new Request.Builder().url(toAbsoluteEndPoint(appUrl, endPoint)); if (token != null) { - url.addHeader(devopsPlatformHeaders.getAuthorizationHeader(), token.getAuthorizationHeaderPrefix() + " " + token); + url.addHeader(devopsPlatformHeaders.getAuthorizationHeader(), token.getAuthorizationHeaderPrefix() + " " + token.getValue()); devopsPlatformHeaders.getApiVersion().ifPresent(apiVersion -> url.addHeader(devopsPlatformHeaders.getApiVersionHeader().orElseThrow(), apiVersion) ); @@ -224,17 +223,16 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli @CheckForNull private static String readNextEndPoint(okhttp3.Response response) { - String links = response.headers().get("link"); - if (links == null || links.isEmpty() || !links.contains("rel=\"next\"")) { - return null; - } - + String links = Optional.ofNullable(response.headers().get("link")).orElse(""); Matcher nextLinkMatcher = NEXT_LINK_PATTERN.matcher(links); if (!nextLinkMatcher.find()) { return null; } - - return nextLinkMatcher.group(1); + String nextUrl = nextLinkMatcher.group(1); + if (response.request().url().toString().equals(nextUrl)) { + return null; + } + return nextUrl; } @CheckForNull @@ -250,7 +248,7 @@ public abstract class GenericApplicationHttpClient implements ApplicationHttpCli @CheckForNull private static <T> T headerValueOrNull(okhttp3.Response response, String header, Function<String, T> mapper) { - return ofNullable(response.header(header)).map(mapper::apply).orElse(null); + return ofNullable(response.headers().get(header)).map(mapper::apply).orElse(null); } private static class ResponseImpl implements Response { diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericPaginatedHttpClient.java new file mode 100644 index 00000000000..51e948318ef --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/GenericPaginatedHttpClient.java @@ -0,0 +1,98 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.alm.client.ApplicationHttpClient.GetResponse; +import org.sonar.alm.client.github.security.AccessToken; + +import static java.lang.String.format; + +public abstract class GenericPaginatedHttpClient implements PaginatedHttpClient { + + private static final Logger LOG = LoggerFactory.getLogger(GenericPaginatedHttpClient.class); + private final ApplicationHttpClient appHttpClient; + private final RatioBasedRateLimitChecker rateLimitChecker; + + protected GenericPaginatedHttpClient(ApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + this.appHttpClient = appHttpClient; + this.rateLimitChecker = rateLimitChecker; + } + + @Override + public <E> List<E> get(String appUrl, AccessToken token, String query, Function<String, List<E>> responseDeserializer) { + List<E> results = new ArrayList<>(); + String nextEndpoint = query + "?per_page=100"; + if (query.contains("?")) { + nextEndpoint = query + "&per_page=100"; + } + ApplicationHttpClient.RateLimit rateLimit = null; + while (nextEndpoint != null) { + checkRateLimit(rateLimit); + GetResponse response = executeCall(appUrl, token, nextEndpoint); + response.getContent() + .ifPresent(content -> results.addAll(responseDeserializer.apply(content))); + nextEndpoint = response.getNextEndPoint().orElse(null); + rateLimit = response.getRateLimit(); + } + return results; + } + + private void checkRateLimit(@Nullable ApplicationHttpClient.RateLimit rateLimit) { + if (rateLimit == null) { + return; + } + try { + rateLimitChecker.checkRateLimit(rateLimit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn(format("Thread interrupted: %s", e.getMessage()), e); + } + } + + private GetResponse executeCall(String appUrl, AccessToken token, String endpoint) { + try { + GetResponse response = appHttpClient.get(appUrl, token, endpoint); + if (response.getCode() < 200 || response.getCode() >= 300) { + throw new IllegalStateException( + format("Error while executing a call to %s. Return code %s. Error message: %s.", appUrl, response.getCode(), response.getContent().orElse(""))); + } + return response; + } catch (Exception e) { + String errorMessage = format("SonarQube was not able to retrieve resources from external system. Error while executing a paginated call to %s, endpoint:%s.", + appUrl, endpoint); + logException(errorMessage, e); + throw new IllegalStateException(errorMessage + " " + e.getMessage()); + } + } + + private static void logException(String message, Exception e) { + if (LOG.isDebugEnabled()) { + LOG.warn(message, e); + } else { + LOG.warn(message, e.getMessage()); + } + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/PaginatedHttpClient.java index 134e942671e..5e746e5709a 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/PaginatedHttpClient.java @@ -17,14 +17,12 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; -import java.io.IOException; import java.util.List; import java.util.function.Function; import org.sonar.alm.client.github.security.AccessToken; public interface PaginatedHttpClient { - - <E> List<E> get(String appUrl, AccessToken token, String query, Function<String, List<E>> responseDeserializer) throws IOException; + <E> List<E> get(String appUrl, AccessToken token, String query, Function<String, List<E>> responseDeserializer); } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/RatioBasedRateLimitChecker.java index 9eb6e46e493..01beeac4489 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/RatioBasedRateLimitChecker.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.alm.client.github; +package org.sonar.alm.client; import com.google.common.annotations.VisibleForTesting; import org.kohsuke.github.GHRateLimit; @@ -33,7 +33,7 @@ public class RatioBasedRateLimitChecker extends RateLimitChecker { private static final Logger LOGGER = LoggerFactory.getLogger(RatioBasedRateLimitChecker.class); @VisibleForTesting - static final String RATE_RATIO_EXCEEDED_MESSAGE = "The GitHub API rate limit is almost reached. Pausing GitHub provisioning until the next rate limit reset. " + static final String RATE_RATIO_EXCEEDED_MESSAGE = "The external system API rate limit is almost reached. Pausing GitHub provisioning until the next rate limit reset. " + "{} out of {} calls were used."; private static final int MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING = 90; @@ -42,7 +42,7 @@ public class RatioBasedRateLimitChecker extends RateLimitChecker { int limit = rateLimitRecord.limit(); int apiCallsUsed = limit - rateLimitRecord.remaining(); double percentageOfCallsUsed = computePercentageOfCallsUsed(apiCallsUsed, limit); - LOGGER.debug("{} GitHub API calls used of {} available per hours", apiCallsUsed, limit); + LOGGER.debug("{} external system API calls used of {}", apiCallsUsed, limit); if (percentageOfCallsUsed >= MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING) { LOGGER.warn(RATE_RATIO_EXCEEDED_MESSAGE, apiCallsUsed, limit); GHRateLimit.Record rateLimit = new GHRateLimit.Record(rateLimitRecord.limit(), rateLimitRecord.remaining(), rateLimitRecord.reset()); diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java index 2e657f7f3f4..d079dc6fb27 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java @@ -37,7 +37,8 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; +import org.sonar.alm.client.ApplicationHttpClient; +import org.sonar.alm.client.ApplicationHttpClient.GetResponse; import org.sonar.alm.client.github.GithubBinding.GsonGithubRepository; import org.sonar.alm.client.github.GithubBinding.GsonInstallations; import org.sonar.alm.client.github.GithubBinding.GsonRepositorySearch; @@ -69,21 +70,17 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { protected static final String WRITE_PERMISSION_NAME = "write"; protected static final String READ_PERMISSION_NAME = "read"; protected static final String FAILED_TO_REQUEST_BEGIN_MSG = "Failed to request "; - - private static final String EXCEPTION_MESSAGE = "SonarQube was not able to retrieve resources from GitHub. " - + "This is likely due to a connectivity problem or a temporary network outage"; - private static final Type REPOSITORY_TEAM_LIST_TYPE = TypeToken.getParameterized(List.class, GsonRepositoryTeam.class).getType(); private static final Type REPOSITORY_COLLABORATORS_LIST_TYPE = TypeToken.getParameterized(List.class, GsonRepositoryCollaborator.class).getType(); private static final Type ORGANIZATION_LIST_TYPE = TypeToken.getParameterized(List.class, GithubBinding.GsonInstallation.class).getType(); - protected final ApplicationHttpClient appHttpClient; + protected final GithubApplicationHttpClient githubApplicationHttpClient; protected final GithubAppSecurity appSecurity; private final GitHubSettings gitHubSettings; - private final PaginatedHttpClient githubPaginatedHttpClient; + private final GithubPaginatedHttpClient githubPaginatedHttpClient; - public GithubApplicationClientImpl(ApplicationHttpClient appHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings, - PaginatedHttpClient githubPaginatedHttpClient) { - this.appHttpClient = appHttpClient; + public GithubApplicationClientImpl(GithubApplicationHttpClient githubApplicationHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings, + GithubPaginatedHttpClient githubPaginatedHttpClient) { + this.githubApplicationHttpClient = githubApplicationHttpClient; this.appSecurity = appSecurity; this.gitHubSettings = gitHubSettings; this.githubPaginatedHttpClient = githubPaginatedHttpClient; @@ -106,7 +103,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { private <T> Optional<T> post(String baseUrl, AccessToken token, String endPoint, Class<T> gsonClass) { try { - ApplicationHttpClient.Response response = appHttpClient.post(baseUrl, token, endPoint); + ApplicationHttpClient.Response response = githubApplicationHttpClient.post(baseUrl, token, endPoint); return handleResponse(response, endPoint, gsonClass); } catch (Exception e) { LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + endPoint, e); @@ -146,7 +143,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { String endPoint = "/app"; GetResponse response; try { - response = appHttpClient.get(githubAppConfiguration.getApiEndpoint(), appToken, endPoint); + response = githubApplicationHttpClient.get(githubAppConfiguration.getApiEndpoint(), appToken, endPoint); } catch (IOException e) { LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + githubAppConfiguration.getApiEndpoint() + endPoint, e); throw new IllegalArgumentException("Failed to validate configuration, check URL and Private Key"); @@ -189,7 +186,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { try { Organizations organizations = new Organizations(); - GetResponse response = appHttpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", page, pageSize)); + GetResponse response = githubApplicationHttpClient.get(appUrl, accessToken, String.format("/user/installations?page=%s&per_page=%s", page, pageSize)); Optional<GsonInstallations> gsonInstallations = response.getContent().map(content -> GSON.fromJson(content, GsonInstallations.class)); if (!gsonInstallations.isPresent()) { @@ -247,7 +244,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { protected <T> Optional<T> get(String baseUrl, AccessToken token, String endPoint, Class<T> gsonClass) { try { - GetResponse response = appHttpClient.get(baseUrl, token, endPoint); + GetResponse response = githubApplicationHttpClient.get(baseUrl, token, endPoint); return handleResponse(response, endPoint, gsonClass); } catch (Exception e) { LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + endPoint, e); @@ -264,7 +261,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { } try { Repositories repositories = new Repositories(); - GetResponse response = appHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", searchQuery, page, pageSize)); + GetResponse response = githubApplicationHttpClient.get(appUrl, accessToken, String.format("/search/repositories?q=%s&page=%s&per_page=%s", searchQuery, page, pageSize)); Optional<GsonRepositorySearch> gsonRepositories = response.getContent().map(content -> GSON.fromJson(content, GsonRepositorySearch.class)); if (!gsonRepositories.isPresent()) { return repositories; @@ -288,7 +285,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { @Override public Optional<Repository> getRepository(String appUrl, AccessToken accessToken, String organizationAndRepository) { try { - GetResponse response = appHttpClient.get(appUrl, accessToken, String.format("/repos/%s", organizationAndRepository)); + GetResponse response = githubApplicationHttpClient.get(appUrl, accessToken, String.format("/repos/%s", organizationAndRepository)); return Optional.of(response) .filter(r -> r.getCode() == HTTP_OK) .flatMap(ApplicationHttpClient.Response::getContent) @@ -315,7 +312,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { baseAppUrl = appUrl; } - ApplicationHttpClient.Response response = appHttpClient.post(baseAppUrl, null, endpoint); + ApplicationHttpClient.Response response = githubApplicationHttpClient.post(baseAppUrl, null, endpoint); if (response.getCode() != HTTP_OK) { throw new IllegalStateException("Failed to create GitHub's user access token. GitHub returned code " + code + ". " + response.getContent().orElse("")); @@ -333,7 +330,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { } // If token is not in the 200's body, it's because the client ID or client secret are incorrect - LOG.error("Failed to create GitHub's user access token. GitHub's response: " + content); + LOG.error("Failed to create GitHub's user access token. GitHub's response: {}", content); throw new IllegalArgumentException(); } catch (IOException e) { throw new IllegalStateException("Failed to create GitHub's user access token", e); @@ -349,7 +346,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { private <T> T getOrThrowIfNotHttpOk(String baseUrl, AccessToken token, String endPoint, Class<T> gsonClass) { try { - GetResponse response = appHttpClient.get(baseUrl, token, endPoint); + GetResponse response = githubApplicationHttpClient.get(baseUrl, token, endPoint); if (response.getCode() != HTTP_OK) { throw new HttpException(baseUrl + endPoint, response.getCode(), response.getContent().orElse("")); } @@ -386,23 +383,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { } private <E> List<E> executePaginatedQuery(String appUrl, AccessToken token, String query, Function<String, List<E>> responseDeserializer) { - try { - return githubPaginatedHttpClient.get(appUrl, token, query, responseDeserializer); - } catch (IOException ioException) { - throw logAndCreateException(ioException, format("Error while executing a paginated call to GitHub - appUrl: %s, path: %s.", appUrl, query)); - } + return githubPaginatedHttpClient.get(appUrl, token, query, responseDeserializer); } - private static IllegalStateException logAndCreateException(IOException ioException, String errorMessage) { - log(errorMessage, ioException); - return new IllegalStateException(EXCEPTION_MESSAGE + ": " + errorMessage + " " + ioException.getMessage()); - } - - private static void log(String message, Exception e) { - if (LOG.isDebugEnabled()) { - LOG.warn(message, e); - } else { - LOG.warn(message); - } - } } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java index 49406cea9b0..24556e3da72 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java @@ -19,6 +19,7 @@ */ package org.sonar.alm.client.github; +import org.sonar.alm.client.GenericApplicationHttpClient; import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java index 9496f0b2bcf..847cf537507 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java @@ -20,6 +20,7 @@ package org.sonar.alm.client.github; import java.util.Optional; +import org.sonar.alm.client.DevopsPlatformHeaders; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java index 9c8d336e192..a68f0cf5d54 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java @@ -19,69 +19,17 @@ */ package org.sonar.alm.client.github; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.alm.client.GenericPaginatedHttpClient; +import org.sonar.alm.client.RatioBasedRateLimitChecker; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; -import static java.lang.String.format; - @ServerSide @ComputeEngineSide -public class GithubPaginatedHttpClient implements PaginatedHttpClient { - - private static final Logger LOG = LoggerFactory.getLogger(GithubPaginatedHttpClient.class); - private final ApplicationHttpClient appHttpClient; - private final RatioBasedRateLimitChecker rateLimitChecker; - - public GithubPaginatedHttpClient(ApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { - this.appHttpClient = appHttpClient; - this.rateLimitChecker = rateLimitChecker; - } +public class GithubPaginatedHttpClient extends GenericPaginatedHttpClient { - @Override - public <E> List<E> get(String appUrl, AccessToken token, String query, Function<String, List<E>> responseDeserializer) throws IOException { - List<E> results = new ArrayList<>(); - String nextEndpoint = query + "?per_page=100"; - if (query.contains("?")) { - nextEndpoint = query + "&per_page=100"; - } - ApplicationHttpClient.RateLimit rateLimit = null; - while (nextEndpoint != null) { - checkRateLimit(rateLimit); - ApplicationHttpClient.GetResponse response = executeCall(appUrl, token, nextEndpoint); - response.getContent() - .ifPresent(content -> results.addAll(responseDeserializer.apply(content))); - nextEndpoint = response.getNextEndPoint().orElse(null); - rateLimit = response.getRateLimit(); - } - return results; + public GithubPaginatedHttpClient(GithubApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + super(appHttpClient, rateLimitChecker); } - private void checkRateLimit(@Nullable ApplicationHttpClient.RateLimit rateLimit) { - if (rateLimit == null) { - return; - } - try { - rateLimitChecker.checkRateLimit(rateLimit); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn(format("Thread interrupted: %s", e.getMessage()), e); - } - } - - private ApplicationHttpClient.GetResponse executeCall(String appUrl, AccessToken token, String endpoint) throws IOException { - ApplicationHttpClient.GetResponse response = appHttpClient.get(appUrl, token, endpoint); - if (response.getCode() < 200 || response.getCode() >= 300) { - throw new IllegalStateException( - format("Error while executing a call to GitHub. Return code %s. Error message: %s.", response.getCode(), response.getContent().orElse(""))); - } - return response; - } } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationClient.java index b93540c9057..13088aec6e7 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationClient.java @@ -19,15 +19,20 @@ */ package org.sonar.alm.client.gitlab; +import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Type; import java.net.URLEncoder; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import javax.annotation.Nullable; import okhttp3.Headers; import okhttp3.OkHttpClient; @@ -39,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.api.server.ServerSide; +import org.sonar.auth.gitlab.GsonGroup; import org.sonarqube.ws.MediaTypes; import org.sonarqube.ws.client.OkHttpClientBuilder; @@ -47,13 +53,18 @@ import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; import static java.nio.charset.StandardCharsets.UTF_8; @ServerSide -public class GitlabHttpClient { +public class GitlabApplicationClient { + private static final Logger LOG = LoggerFactory.getLogger(GitlabApplicationClient.class); + private static final Gson GSON = new Gson(); + private static final Type GITLAB_GROUP = TypeToken.getParameterized(List.class, GsonGroup.class).getType(); - private static final Logger LOG = LoggerFactory.getLogger(GitlabHttpClient.class); protected static final String PRIVATE_TOKEN = "Private-Token"; protected final OkHttpClient client; - public GitlabHttpClient(TimeoutConfiguration timeoutConfiguration) { + private final GitlabPaginatedHttpClient gitlabPaginatedHttpClient; + + public GitlabApplicationClient(GitlabPaginatedHttpClient gitlabPaginatedHttpClient, TimeoutConfiguration timeoutConfiguration) { + this.gitlabPaginatedHttpClient = gitlabPaginatedHttpClient; client = new OkHttpClientBuilder() .setConnectTimeoutMs(timeoutConfiguration.getConnectTimeout()) .setReadTimeoutMs(timeoutConfiguration.getReadTimeout()) @@ -324,34 +335,6 @@ public class GitlabHttpClient { } } - /*public void getGroups(String gitlabUrl, String token) { - String url = String.format("%s/groups", gitlabUrl); - LOG.debug(String.format("get groups : [%s]", url)); - - Request request = new Request.Builder() - .addHeader(PRIVATE_TOKEN, token) - .url(url) - .get() - .build(); - - - try (Response response = client.newCall(request).execute()) { - Headers headers = response.headers(); - checkResponseIsSuccessful(response, "Could not get projects from GitLab instance"); - List<Project> projectList = Project.parseJsonArray(response.body().string()); - int returnedPageNumber = parseAndGetIntegerHeader(headers.get("X-Page")); - int returnedPageSize = parseAndGetIntegerHeader(headers.get("X-Per-Page")); - String xtotal = headers.get("X-Total"); - Integer totalProjects = Strings.isEmpty(xtotal) ? null : parseAndGetIntegerHeader(xtotal); - return new ProjectList(projectList, returnedPageNumber, returnedPageSize, totalProjects); - } catch (JsonSyntaxException e) { - throw new IllegalArgumentException("Could not parse GitLab answer to search projects. Got a non-json payload as result."); - } catch (IOException e) { - logException(url, e); - throw new IllegalStateException(e.getMessage(), e); - } - }*/ - private static int parseAndGetIntegerHeader(@Nullable String header) { if (header == null) { throw new IllegalArgumentException("Pagination data from GitLab response is missing"); @@ -364,4 +347,13 @@ public class GitlabHttpClient { } } + public Set<GsonGroup> getGroups(String gitlabUrl, String token) { + return Set.copyOf(executePaginatedQuery(gitlabUrl, token, "/groups", resp -> GSON.fromJson(resp, GITLAB_GROUP))); + } + + private <E> List<E> executePaginatedQuery(String appUrl, String token, String query, Function<String, List<E>> responseDeserializer) { + GitlabToken gitlabToken = new GitlabToken(token); + return gitlabPaginatedHttpClient.get(appUrl, gitlabToken, query, responseDeserializer); + } + } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationHttpClient.java new file mode 100644 index 00000000000..ec9c15f8673 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabApplicationHttpClient.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import org.sonar.alm.client.TimeoutConfiguration; +import org.sonar.alm.client.GenericApplicationHttpClient; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public class GitlabApplicationHttpClient extends GenericApplicationHttpClient { + public GitlabApplicationHttpClient(GitlabHeaders gitlabHeaders, TimeoutConfiguration timeoutConfiguration) { + super(gitlabHeaders, timeoutConfiguration); + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java index 2ca75b19e26..c1f76a15274 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabGlobalSettingsValidator.java @@ -28,11 +28,11 @@ import org.sonar.db.alm.setting.AlmSettingDto; public class GitlabGlobalSettingsValidator { private final Encryption encryption; - private final GitlabHttpClient gitlabHttpClient; + private final GitlabApplicationClient gitlabApplicationClient; - public GitlabGlobalSettingsValidator(GitlabHttpClient gitlabHttpClient, Settings settings) { + public GitlabGlobalSettingsValidator(GitlabApplicationClient gitlabApplicationClient, Settings settings) { this.encryption = settings.getEncryption(); - this.gitlabHttpClient = gitlabHttpClient; + this.gitlabApplicationClient = gitlabApplicationClient; } public void validate(AlmSettingDto almSettingDto) { @@ -43,10 +43,10 @@ public class GitlabGlobalSettingsValidator { throw new IllegalArgumentException("Your Gitlab global configuration is incomplete."); } - gitlabHttpClient.checkUrl(gitlabUrl); - gitlabHttpClient.checkToken(gitlabUrl, accessToken); - gitlabHttpClient.checkReadPermission(gitlabUrl, accessToken); - gitlabHttpClient.checkWritePermission(gitlabUrl, accessToken); + gitlabApplicationClient.checkUrl(gitlabUrl); + gitlabApplicationClient.checkToken(gitlabUrl, accessToken); + gitlabApplicationClient.checkReadPermission(gitlabUrl, accessToken); + gitlabApplicationClient.checkWritePermission(gitlabUrl, accessToken); } } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHeaders.java new file mode 100644 index 00000000000..e09d7446e8a --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHeaders.java @@ -0,0 +1,60 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import java.util.Optional; +import org.sonar.alm.client.DevopsPlatformHeaders; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public class GitlabHeaders implements DevopsPlatformHeaders { + + @Override + public Optional<String> getApiVersionHeader() { + return Optional.empty(); + } + + @Override + public Optional<String> getApiVersion() { + return Optional.empty(); + } + + @Override + public String getRateLimitRemainingHeader() { + return "ratelimit-remaining"; + } + + @Override + public String getRateLimitLimitHeader() { + return "ratelimit-limit"; + } + + @Override + public String getRateLimitResetHeader() { + return "ratelimit-reset"; + } + + @Override + public String getAuthorizationHeader() { + return "PRIVATE-TOKEN"; + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabPaginatedHttpClient.java new file mode 100644 index 00000000000..0e02c84c53c --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabPaginatedHttpClient.java @@ -0,0 +1,35 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import org.sonar.alm.client.GenericPaginatedHttpClient; +import org.sonar.alm.client.RatioBasedRateLimitChecker; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public class GitlabPaginatedHttpClient extends GenericPaginatedHttpClient { + + public GitlabPaginatedHttpClient(GitlabApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + super(appHttpClient, rateLimitChecker); + } + +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabToken.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabToken.java new file mode 100644 index 00000000000..c6c3c0a3112 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabToken.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.gitlab; + +import java.util.Objects; +import org.sonar.alm.client.github.security.AccessToken; + +public class GitlabToken implements AccessToken { + private final String token; + + public GitlabToken(String token) { + this.token = token; + } + + @Override + public String getValue() { + return token; + } + + @Override + public String getAuthorizationHeaderPrefix() { + return ""; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + GitlabToken that = (GitlabToken) o; + return Objects.equals(token, that.token); + } + + @Override + public int hashCode() { + return Objects.hash(token); + } +} |