From: Aurelien Poscia Date: Wed, 29 Nov 2023 15:17:57 +0000 (+0100) Subject: SONAR-21119 made GithubApplicationHttpClient generic and as well as rate checking... X-Git-Tag: 10.4.0.87286~273 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=c0c9226eb421d0581b269c74a083aad00a7ad679;p=sonarqube.git SONAR-21119 made GithubApplicationHttpClient generic and as well as rate checking mechanism --- diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java new file mode 100644 index 00000000000..75c512eddb5 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/ApplicationHttpClient.java @@ -0,0 +1,93 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import java.io.IOException; +import java.util.Optional; +import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public interface ApplicationHttpClient { + /** + * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. + */ + GetResponse get(String appUrl, AccessToken token, String endPoint) throws IOException; + + /** + * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. + * No log if there is an issue during the call. + */ + GetResponse getSilent(String appUrl, AccessToken token, String endPoint) throws IOException; + + /** + * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK} or + * {@link java.net.HttpURLConnection#HTTP_CREATED CREATED}. + */ + Response post(String appUrl, AccessToken token, String endPoint) throws IOException; + + /** + * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK} or + * {@link java.net.HttpURLConnection#HTTP_CREATED CREATED}. + * + * Content type will be application/json; charset=utf-8 + */ + Response post(String appUrl, AccessToken token, String endPoint, String json) throws IOException; + + /** + * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. + * + * Content type will be application/json; charset=utf-8 + */ + Response patch(String appUrl, AccessToken token, String endPoint, String json) throws IOException; + + /** + * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. + * + * Content type will be application/json; charset=utf-8 + * + */ + Response delete(String appUrl, AccessToken token, String endPoint) throws IOException; + + record RateLimit(int remaining, int limit, long reset) { + } + interface Response { + + /** + * @return the HTTP code of the response. + */ + int getCode(); + + /** + * @return the content of the response if the response had an HTTP code for which we expect a content for the current + * HTTP method (see {@link #get(String, AccessToken, String)} and {@link #post(String, AccessToken, String)}). + */ + Optional getContent(); + + RateLimit getRateLimit(); + } + + interface GetResponse extends Response { + Optional getNextEndPoint(); + } + +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java new file mode 100644 index 00000000000..38e8fe0b94c --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/DevopsPlatformHeaders.java @@ -0,0 +1,36 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import java.util.Optional; + +public interface DevopsPlatformHeaders { + Optional getApiVersionHeader(); + + Optional getApiVersion(); + + String getRateLimitRemainingHeader(); + + String getRateLimitLimitHeader(); + + String getRateLimitResetHeader(); + + String getAuthorizationHeader(); +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java new file mode 100644 index 00000000000..d67224c2615 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GenericApplicationHttpClient.java @@ -0,0 +1,299 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import okhttp3.FormBody; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.ResponseBody; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.alm.client.TimeoutConfiguration; +import org.sonar.alm.client.github.security.AccessToken; +import org.sonarqube.ws.client.OkHttpClientBuilder; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.net.HttpURLConnection.HTTP_ACCEPTED; +import static java.net.HttpURLConnection.HTTP_CREATED; +import static java.net.HttpURLConnection.HTTP_NO_CONTENT; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.util.Optional.empty; +import static java.util.Optional.of; +import static java.util.Optional.ofNullable; + +public abstract class GenericApplicationHttpClient implements ApplicationHttpClient { + + private static final Logger LOG = LoggerFactory.getLogger(GenericApplicationHttpClient.class); + private static final Pattern NEXT_LINK_PATTERN = Pattern.compile("<([^<]+)>; rel=\"next\""); + + private final DevopsPlatformHeaders devopsPlatformHeaders; + private final OkHttpClient client; + + public GenericApplicationHttpClient(DevopsPlatformHeaders devopsPlatformHeaders, TimeoutConfiguration timeoutConfiguration) { + this.devopsPlatformHeaders = devopsPlatformHeaders; + client = new OkHttpClientBuilder() + .setConnectTimeoutMs(timeoutConfiguration.getConnectTimeout()) + .setReadTimeoutMs(timeoutConfiguration.getReadTimeout()) + .setFollowRedirects(false) + .build(); + } + + @Override + public GetResponse get(String appUrl, AccessToken token, String endPoint) throws IOException { + return get(appUrl, token, endPoint, true); + } + + @Override + public GetResponse getSilent(String appUrl, AccessToken token, String endPoint) throws IOException { + return get(appUrl, token, endPoint, false); + } + + private GetResponse get(String appUrl, AccessToken token, String endPoint, boolean withLog) throws IOException { + validateEndPoint(endPoint); + try (okhttp3.Response response = client.newCall(newGetRequest(appUrl, token, endPoint)).execute()) { + int responseCode = response.code(); + RateLimit rateLimit = readRateLimit(response); + if (responseCode != HTTP_OK) { + String content = StringUtils.trimToNull(attemptReadContent(response)); + if (withLog) { + LOG.warn("GET response did not have expected HTTP code (was {}): {}", responseCode, content); + } + return new GetResponseImpl(responseCode, content, null, rateLimit); + } + return new GetResponseImpl(responseCode, readContent(response.body()).orElse(null), readNextEndPoint(response), rateLimit); + } + } + + private static void validateEndPoint(String endPoint) { + checkArgument(endPoint.startsWith("/") || endPoint.startsWith("http") || endPoint.isEmpty(), + "endpoint must start with '/' or 'http'"); + } + + private Request newGetRequest(String appUrl, AccessToken token, String endPoint) { + return newRequestBuilder(appUrl, token, endPoint).get().build(); + } + + @Override + public Response post(String appUrl, AccessToken token, String endPoint) throws IOException { + return doPost(appUrl, token, endPoint, new FormBody.Builder().build()); + } + + @Override + public Response post(String appUrl, AccessToken token, String endPoint, String json) throws IOException { + RequestBody body = RequestBody.create(json, MediaType.parse("application/json; charset=utf-8")); + return doPost(appUrl, token, endPoint, body); + } + + @Override + public Response patch(String appUrl, AccessToken token, String endPoint, String json) throws IOException { + RequestBody body = RequestBody.create(json, MediaType.parse("application/json; charset=utf-8")); + return doPatch(appUrl, token, endPoint, body); + } + + @Override + public Response delete(String appUrl, AccessToken token, String endPoint) throws IOException { + validateEndPoint(endPoint); + + try (okhttp3.Response response = client.newCall(newDeleteRequest(appUrl, token, endPoint)).execute()) { + int responseCode = response.code(); + RateLimit rateLimit = readRateLimit(response); + if (responseCode != HTTP_NO_CONTENT) { + String content = attemptReadContent(response); + LOG.warn("DELETE response did not have expected HTTP code (was {}): {}", responseCode, content); + return new ResponseImpl(responseCode, content, rateLimit); + } + return new ResponseImpl(responseCode, null, rateLimit); + } + } + + private Request newDeleteRequest(String appUrl, AccessToken token, String endPoint) { + return newRequestBuilder(appUrl, token, endPoint).delete().build(); + } + + private Response doPost(String appUrl, @Nullable AccessToken token, String endPoint, RequestBody body) throws IOException { + validateEndPoint(endPoint); + + try (okhttp3.Response response = client.newCall(newPostRequest(appUrl, token, endPoint, body)).execute()) { + int responseCode = response.code(); + RateLimit rateLimit = readRateLimit(response); + if (responseCode == HTTP_OK || responseCode == HTTP_CREATED || responseCode == HTTP_ACCEPTED) { + return new ResponseImpl(responseCode, readContent(response.body()).orElse(null), rateLimit); + } else if (responseCode == HTTP_NO_CONTENT) { + return new ResponseImpl(responseCode, null, rateLimit); + } + String content = attemptReadContent(response); + LOG.warn("POST response did not have expected HTTP code (was {}): {}", responseCode, content); + return new ResponseImpl(responseCode, content, rateLimit); + } + } + + private Response doPatch(String appUrl, AccessToken token, String endPoint, RequestBody body) throws IOException { + validateEndPoint(endPoint); + + try (okhttp3.Response response = client.newCall(newPatchRequest(token, appUrl, endPoint, body)).execute()) { + int responseCode = response.code(); + RateLimit rateLimit = readRateLimit(response); + if (responseCode == HTTP_OK) { + return new ResponseImpl(responseCode, readContent(response.body()).orElse(null), rateLimit); + } else if (responseCode == HTTP_NO_CONTENT) { + return new ResponseImpl(responseCode, null, rateLimit); + } + String content = attemptReadContent(response); + LOG.warn("PATCH response did not have expected HTTP code (was {}): {}", responseCode, content); + return new ResponseImpl(responseCode, content, rateLimit); + } + } + + private Request newPostRequest(String appUrl, @Nullable AccessToken token, String endPoint, RequestBody body) { + return newRequestBuilder(appUrl, token, endPoint).post(body).build(); + } + + private Request newPatchRequest(AccessToken token, String appUrl, String endPoint, RequestBody body) { + return newRequestBuilder(appUrl, token, endPoint).patch(body).build(); + } + + private Request.Builder newRequestBuilder(String appUrl, @Nullable AccessToken token, String endPoint) { + Request.Builder url = new Request.Builder().url(toAbsoluteEndPoint(appUrl, endPoint)); + if (token != null) { + url.addHeader(devopsPlatformHeaders.getAuthorizationHeader(), token.getAuthorizationHeaderPrefix() + " " + token); + devopsPlatformHeaders.getApiVersion().ifPresent(apiVersion -> + url.addHeader(devopsPlatformHeaders.getApiVersionHeader().orElseThrow(), apiVersion) + ); + } + return url; + } + + private static String toAbsoluteEndPoint(String host, String endPoint) { + if (endPoint.startsWith("http")) { + return endPoint; + } + try { + return new URL(host + endPoint).toExternalForm(); + } catch (MalformedURLException e) { + throw new IllegalArgumentException(String.format("%s is not a valid url", host + endPoint)); + } + } + + private static String attemptReadContent(okhttp3.Response response) { + try { + return readContent(response.body()).orElse(null); + } catch (IOException e) { + return null; + } + } + + private static Optional readContent(@Nullable ResponseBody body) throws IOException { + if (body == null) { + return empty(); + } + try { + return of(body.string()); + } finally { + body.close(); + } + } + + @CheckForNull + private static String readNextEndPoint(okhttp3.Response response) { + String links = response.headers().get("link"); + if (links == null || links.isEmpty() || !links.contains("rel=\"next\"")) { + return null; + } + + Matcher nextLinkMatcher = NEXT_LINK_PATTERN.matcher(links); + if (!nextLinkMatcher.find()) { + return null; + } + + return nextLinkMatcher.group(1); + } + + @CheckForNull + private RateLimit readRateLimit(okhttp3.Response response) { + Integer remaining = headerValueOrNull(response, devopsPlatformHeaders.getRateLimitRemainingHeader(), Integer::valueOf); + Integer limit = headerValueOrNull(response, devopsPlatformHeaders.getRateLimitLimitHeader(), Integer::valueOf); + Long reset = headerValueOrNull(response, devopsPlatformHeaders.getRateLimitResetHeader(), Long::valueOf); + if (remaining == null || limit == null || reset == null) { + return null; + } + return new RateLimit(remaining, limit, reset); + } + + @CheckForNull + private static T headerValueOrNull(okhttp3.Response response, String header, Function mapper) { + return ofNullable(response.header(header)).map(mapper::apply).orElse(null); + } + + private static class ResponseImpl implements Response { + private final int code; + private final String content; + + private final RateLimit rateLimit; + + private ResponseImpl(int code, @Nullable String content, @Nullable RateLimit rateLimit) { + this.code = code; + this.content = content; + this.rateLimit = rateLimit; + } + + @Override + public int getCode() { + return code; + } + + @Override + public Optional getContent() { + return ofNullable(content); + } + + @Override + @CheckForNull + public RateLimit getRateLimit() { + return rateLimit; + } + + } + + private static final class GetResponseImpl extends ResponseImpl implements GetResponse { + private final String nextEndPoint; + + private GetResponseImpl(int code, @Nullable String content, @Nullable String nextEndPoint, @Nullable RateLimit rateLimit) { + super(code, content, rateLimit); + this.nextEndPoint = nextEndPoint; + } + + @Override + public Optional getNextEndPoint() { + return ofNullable(nextEndPoint); + } + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java index 477f6710741..2e657f7f3f4 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java @@ -37,7 +37,7 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.alm.client.github.GithubApplicationHttpClient.GetResponse; +import org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; import org.sonar.alm.client.github.GithubBinding.GsonGithubRepository; import org.sonar.alm.client.github.GithubBinding.GsonInstallations; import org.sonar.alm.client.github.GithubBinding.GsonRepositorySearch; @@ -76,13 +76,13 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { private static final Type REPOSITORY_TEAM_LIST_TYPE = TypeToken.getParameterized(List.class, GsonRepositoryTeam.class).getType(); private static final Type REPOSITORY_COLLABORATORS_LIST_TYPE = TypeToken.getParameterized(List.class, GsonRepositoryCollaborator.class).getType(); private static final Type ORGANIZATION_LIST_TYPE = TypeToken.getParameterized(List.class, GithubBinding.GsonInstallation.class).getType(); - protected final GithubApplicationHttpClient appHttpClient; + protected final ApplicationHttpClient appHttpClient; protected final GithubAppSecurity appSecurity; private final GitHubSettings gitHubSettings; - private final GithubPaginatedHttpClient githubPaginatedHttpClient; + private final PaginatedHttpClient githubPaginatedHttpClient; - public GithubApplicationClientImpl(GithubApplicationHttpClient appHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings, - GithubPaginatedHttpClient githubPaginatedHttpClient) { + public GithubApplicationClientImpl(ApplicationHttpClient appHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings, + PaginatedHttpClient githubPaginatedHttpClient) { this.appHttpClient = appHttpClient; this.appSecurity = appSecurity; this.gitHubSettings = gitHubSettings; @@ -106,7 +106,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { private Optional post(String baseUrl, AccessToken token, String endPoint, Class gsonClass) { try { - GithubApplicationHttpClient.Response response = appHttpClient.post(baseUrl, token, endPoint); + ApplicationHttpClient.Response response = appHttpClient.post(baseUrl, token, endPoint); return handleResponse(response, endPoint, gsonClass); } catch (Exception e) { LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + endPoint, e); @@ -291,7 +291,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { GetResponse response = appHttpClient.get(appUrl, accessToken, String.format("/repos/%s", organizationAndRepository)); return Optional.of(response) .filter(r -> r.getCode() == HTTP_OK) - .flatMap(GithubApplicationHttpClient.Response::getContent) + .flatMap(ApplicationHttpClient.Response::getContent) .map(content -> GSON.fromJson(content, GsonGithubRepository.class)) .map(GsonGithubRepository::toRepository); } catch (Exception e) { @@ -315,7 +315,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { baseAppUrl = appUrl; } - GithubApplicationHttpClient.Response response = appHttpClient.post(baseAppUrl, null, endpoint); + ApplicationHttpClient.Response response = appHttpClient.post(baseAppUrl, null, endpoint); if (response.getCode() != HTTP_OK) { throw new IllegalStateException("Failed to create GitHub's user access token. GitHub returned code " + code + ". " + response.getContent().orElse("")); @@ -359,7 +359,7 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { } } - protected static Optional handleResponse(GithubApplicationHttpClient.Response response, String endPoint, Class gsonClass) { + protected static Optional handleResponse(ApplicationHttpClient.Response response, String endPoint, Class gsonClass) { try { return response.getContent().map(c -> GSON.fromJson(c, gsonClass)); } catch (Exception e) { diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java index 10c99ee84c3..49406cea9b0 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClient.java @@ -19,75 +19,14 @@ */ package org.sonar.alm.client.github; -import java.io.IOException; -import java.util.Optional; -import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.alm.client.TimeoutConfiguration; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; @ServerSide @ComputeEngineSide -public interface GithubApplicationHttpClient { - /** - * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. - */ - GetResponse get(String appUrl, AccessToken token, String endPoint) throws IOException; - - /** - * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. - * No log if there is an issue during the call. - */ - GetResponse getSilent(String appUrl, AccessToken token, String endPoint) throws IOException; - - /** - * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK} or - * {@link java.net.HttpURLConnection#HTTP_CREATED CREATED}. - */ - Response post(String appUrl, AccessToken token, String endPoint) throws IOException; - - /** - * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK} or - * {@link java.net.HttpURLConnection#HTTP_CREATED CREATED}. - * - * Content type will be application/json; charset=utf-8 - */ - Response post(String appUrl, AccessToken token, String endPoint, String json) throws IOException; - - /** - * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. - * - * Content type will be application/json; charset=utf-8 - */ - Response patch(String appUrl, AccessToken token, String endPoint, String json) throws IOException; - - /** - * Content of the response is populated if response's HTTP code is {@link java.net.HttpURLConnection#HTTP_OK OK}. - * - * Content type will be application/json; charset=utf-8 - * - */ - Response delete(String appUrl, AccessToken token, String endPoint) throws IOException; - - record RateLimit(int remaining, int limit, long reset) { +public class GithubApplicationHttpClient extends GenericApplicationHttpClient { + public GithubApplicationHttpClient(GithubHeaders githubHeaders, TimeoutConfiguration timeoutConfiguration) { + super(githubHeaders, timeoutConfiguration); } - interface Response { - - /** - * @return the HTTP code of the response. - */ - int getCode(); - - /** - * @return the content of the response if the response had an HTTP code for which we expect a content for the current - * HTTP method (see {@link #get(String, AccessToken, String)} and {@link #post(String, AccessToken, String)}). - */ - Optional getContent(); - - RateLimit getRateLimit(); - } - - interface GetResponse extends Response { - Optional getNextEndPoint(); - } - } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClientImpl.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClientImpl.java deleted file mode 100644 index 5ab475e6e46..00000000000 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationHttpClientImpl.java +++ /dev/null @@ -1,301 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.alm.client.github; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Optional; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; -import okhttp3.FormBody; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.ResponseBody; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.alm.client.TimeoutConfiguration; -import org.sonar.alm.client.github.security.AccessToken; -import org.sonarqube.ws.client.OkHttpClientBuilder; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.net.HttpURLConnection.HTTP_ACCEPTED; -import static java.net.HttpURLConnection.HTTP_CREATED; -import static java.net.HttpURLConnection.HTTP_NO_CONTENT; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.util.Optional.empty; -import static java.util.Optional.of; -import static java.util.Optional.ofNullable; - -public class GithubApplicationHttpClientImpl implements GithubApplicationHttpClient { - - private static final Logger LOG = LoggerFactory.getLogger(GithubApplicationHttpClientImpl.class); - private static final Pattern NEXT_LINK_PATTERN = Pattern.compile("<([^<]+)>; rel=\"next\""); - private static final String GH_API_VERSION_HEADER = "X-GitHub-Api-Version"; - private static final String GH_API_VERSION = "2022-11-28"; - - private static final String GH_RATE_LIMIT_REMAINING_HEADER = "x-ratelimit-remaining"; - private static final String GH_RATE_LIMIT_LIMIT_HEADER = "x-ratelimit-limit"; - private static final String GH_RATE_LIMIT_RESET_HEADER = "x-ratelimit-reset"; - - private final OkHttpClient client; - - public GithubApplicationHttpClientImpl(TimeoutConfiguration timeoutConfiguration) { - client = new OkHttpClientBuilder() - .setConnectTimeoutMs(timeoutConfiguration.getConnectTimeout()) - .setReadTimeoutMs(timeoutConfiguration.getReadTimeout()) - .setFollowRedirects(false) - .build(); - } - - @Override - public GetResponse get(String appUrl, AccessToken token, String endPoint) throws IOException { - return get(appUrl, token, endPoint, true); - } - - @Override - public GetResponse getSilent(String appUrl, AccessToken token, String endPoint) throws IOException { - return get(appUrl, token, endPoint, false); - } - - private GetResponse get(String appUrl, AccessToken token, String endPoint, boolean withLog) throws IOException { - validateEndPoint(endPoint); - try (okhttp3.Response response = client.newCall(newGetRequest(appUrl, token, endPoint)).execute()) { - int responseCode = response.code(); - RateLimit rateLimit = readRateLimit(response); - if (responseCode != HTTP_OK) { - String content = StringUtils.trimToNull(attemptReadContent(response)); - if (withLog) { - LOG.warn("GET response did not have expected HTTP code (was {}): {}", responseCode, content); - } - return new GetResponseImpl(responseCode, content, null, rateLimit); - } - return new GetResponseImpl(responseCode, readContent(response.body()).orElse(null), readNextEndPoint(response), rateLimit); - } - } - - private static void validateEndPoint(String endPoint) { - checkArgument(endPoint.startsWith("/") || endPoint.startsWith("http") || endPoint.isEmpty(), - "endpoint must start with '/' or 'http'"); - } - - private static Request newGetRequest(String appUrl, AccessToken token, String endPoint) { - return newRequestBuilder(appUrl, token, endPoint).get().build(); - } - - @Override - public Response post(String appUrl, AccessToken token, String endPoint) throws IOException { - return doPost(appUrl, token, endPoint, new FormBody.Builder().build()); - } - - @Override - public Response post(String appUrl, AccessToken token, String endPoint, String json) throws IOException { - RequestBody body = RequestBody.create(json, MediaType.parse("application/json; charset=utf-8")); - return doPost(appUrl, token, endPoint, body); - } - - @Override - public Response patch(String appUrl, AccessToken token, String endPoint, String json) throws IOException { - RequestBody body = RequestBody.create(json, MediaType.parse("application/json; charset=utf-8")); - return doPatch(appUrl, token, endPoint, body); - } - - @Override - public Response delete(String appUrl, AccessToken token, String endPoint) throws IOException { - validateEndPoint(endPoint); - - try (okhttp3.Response response = client.newCall(newDeleteRequest(appUrl, token, endPoint)).execute()) { - int responseCode = response.code(); - RateLimit rateLimit = readRateLimit(response); - if (responseCode != HTTP_NO_CONTENT) { - String content = attemptReadContent(response); - LOG.warn("DELETE response did not have expected HTTP code (was {}): {}", responseCode, content); - return new ResponseImpl(responseCode, content, rateLimit); - } - return new ResponseImpl(responseCode, null, rateLimit); - } - } - - private static Request newDeleteRequest(String appUrl, AccessToken token, String endPoint) { - return newRequestBuilder(appUrl, token, endPoint).delete().build(); - } - - private Response doPost(String appUrl, @Nullable AccessToken token, String endPoint, RequestBody body) throws IOException { - validateEndPoint(endPoint); - - try (okhttp3.Response response = client.newCall(newPostRequest(appUrl, token, endPoint, body)).execute()) { - int responseCode = response.code(); - RateLimit rateLimit = readRateLimit(response); - if (responseCode == HTTP_OK || responseCode == HTTP_CREATED || responseCode == HTTP_ACCEPTED) { - return new ResponseImpl(responseCode, readContent(response.body()).orElse(null), rateLimit); - } else if (responseCode == HTTP_NO_CONTENT) { - return new ResponseImpl(responseCode, null, rateLimit); - } - String content = attemptReadContent(response); - LOG.warn("POST response did not have expected HTTP code (was {}): {}", responseCode, content); - return new ResponseImpl(responseCode, content, rateLimit); - } - } - - private Response doPatch(String appUrl, AccessToken token, String endPoint, RequestBody body) throws IOException { - validateEndPoint(endPoint); - - try (okhttp3.Response response = client.newCall(newPatchRequest(token, appUrl, endPoint, body)).execute()) { - int responseCode = response.code(); - RateLimit rateLimit = readRateLimit(response); - if (responseCode == HTTP_OK) { - return new ResponseImpl(responseCode, readContent(response.body()).orElse(null), rateLimit); - } else if (responseCode == HTTP_NO_CONTENT) { - return new ResponseImpl(responseCode, null, rateLimit); - } - String content = attemptReadContent(response); - LOG.warn("PATCH response did not have expected HTTP code (was {}): {}", responseCode, content); - return new ResponseImpl(responseCode, content, rateLimit); - } - } - - private static Request newPostRequest(String appUrl, @Nullable AccessToken token, String endPoint, RequestBody body) { - return newRequestBuilder(appUrl, token, endPoint).post(body).build(); - } - - private static Request newPatchRequest(AccessToken token, String appUrl, String endPoint, RequestBody body) { - return newRequestBuilder(appUrl, token, endPoint).patch(body).build(); - } - - private static Request.Builder newRequestBuilder(String appUrl, @Nullable AccessToken token, String endPoint) { - Request.Builder url = new Request.Builder().url(toAbsoluteEndPoint(appUrl, endPoint)); - if (token != null) { - url.addHeader("Authorization", token.getAuthorizationHeaderPrefix() + " " + token); - url.addHeader(GH_API_VERSION_HEADER, GH_API_VERSION); - } - return url; - } - - private static String toAbsoluteEndPoint(String host, String endPoint) { - if (endPoint.startsWith("http")) { - return endPoint; - } - try { - return new URL(host + endPoint).toExternalForm(); - } catch (MalformedURLException e) { - throw new IllegalArgumentException(String.format("%s is not a valid url", host + endPoint)); - } - } - - private static String attemptReadContent(okhttp3.Response response) { - try { - return readContent(response.body()).orElse(null); - } catch (IOException e) { - return null; - } - } - - private static Optional readContent(@Nullable ResponseBody body) throws IOException { - if (body == null) { - return empty(); - } - try { - return of(body.string()); - } finally { - body.close(); - } - } - - @CheckForNull - private static String readNextEndPoint(okhttp3.Response response) { - String links = response.headers().get("link"); - if (links == null || links.isEmpty() || !links.contains("rel=\"next\"")) { - return null; - } - - Matcher nextLinkMatcher = NEXT_LINK_PATTERN.matcher(links); - if (!nextLinkMatcher.find()) { - return null; - } - - return nextLinkMatcher.group(1); - } - - @CheckForNull - private static RateLimit readRateLimit(okhttp3.Response response) { - Integer remaining = headerValueOrNull(response, GH_RATE_LIMIT_REMAINING_HEADER, Integer::valueOf); - Integer limit = headerValueOrNull(response, GH_RATE_LIMIT_LIMIT_HEADER, Integer::valueOf); - Long reset = headerValueOrNull(response, GH_RATE_LIMIT_RESET_HEADER, Long::valueOf); - if (remaining == null || limit == null || reset == null) { - return null; - } - return new RateLimit(remaining, limit, reset); - } - - @CheckForNull - private static T headerValueOrNull(okhttp3.Response response, String header, Function mapper) { - return ofNullable(response.header(header)).map(mapper::apply).orElse(null); - } - - private static class ResponseImpl implements Response { - private final int code; - private final String content; - - private final RateLimit rateLimit; - - private ResponseImpl(int code, @Nullable String content, @Nullable RateLimit rateLimit) { - this.code = code; - this.content = content; - this.rateLimit = rateLimit; - } - - @Override - public int getCode() { - return code; - } - - @Override - public Optional getContent() { - return ofNullable(content); - } - - @Override - @CheckForNull - public RateLimit getRateLimit() { - return rateLimit; - } - - } - - private static final class GetResponseImpl extends ResponseImpl implements GetResponse { - private final String nextEndPoint; - - private GetResponseImpl(int code, @Nullable String content, @Nullable String nextEndPoint, @Nullable RateLimit rateLimit) { - super(code, content, rateLimit); - this.nextEndPoint = nextEndPoint; - } - - @Override - public Optional getNextEndPoint() { - return ofNullable(nextEndPoint); - } - } -} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java new file mode 100644 index 00000000000..9496f0b2bcf --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubHeaders.java @@ -0,0 +1,59 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import java.util.Optional; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ServerSide +@ComputeEngineSide +public class GithubHeaders implements DevopsPlatformHeaders { + + @Override + public Optional getApiVersionHeader() { + return Optional.of("X-GitHub-Api-Version"); + } + + @Override + public Optional getApiVersion() { + return Optional.of("2022-11-28"); + } + + @Override + public String getRateLimitRemainingHeader() { + return "x-ratelimit-remaining"; + } + + @Override + public String getRateLimitLimitHeader() { + return "x-ratelimit-limit"; + } + + @Override + public String getRateLimitResetHeader() { + return "x-ratelimit-reset"; + } + + @Override + public String getAuthorizationHeader() { + return "Authorization"; + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java index ba4f6379696..9c8d336e192 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java @@ -20,11 +20,68 @@ package org.sonar.alm.client.github; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.function.Function; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; -public interface GithubPaginatedHttpClient { +import static java.lang.String.format; - List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException; +@ServerSide +@ComputeEngineSide +public class GithubPaginatedHttpClient implements PaginatedHttpClient { + + private static final Logger LOG = LoggerFactory.getLogger(GithubPaginatedHttpClient.class); + private final ApplicationHttpClient appHttpClient; + private final RatioBasedRateLimitChecker rateLimitChecker; + + public GithubPaginatedHttpClient(ApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + this.appHttpClient = appHttpClient; + this.rateLimitChecker = rateLimitChecker; + } + + @Override + public List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException { + List results = new ArrayList<>(); + String nextEndpoint = query + "?per_page=100"; + if (query.contains("?")) { + nextEndpoint = query + "&per_page=100"; + } + ApplicationHttpClient.RateLimit rateLimit = null; + while (nextEndpoint != null) { + checkRateLimit(rateLimit); + ApplicationHttpClient.GetResponse response = executeCall(appUrl, token, nextEndpoint); + response.getContent() + .ifPresent(content -> results.addAll(responseDeserializer.apply(content))); + nextEndpoint = response.getNextEndPoint().orElse(null); + rateLimit = response.getRateLimit(); + } + return results; + } + + private void checkRateLimit(@Nullable ApplicationHttpClient.RateLimit rateLimit) { + if (rateLimit == null) { + return; + } + try { + rateLimitChecker.checkRateLimit(rateLimit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn(format("Thread interrupted: %s", e.getMessage()), e); + } + } + + private ApplicationHttpClient.GetResponse executeCall(String appUrl, AccessToken token, String endpoint) throws IOException { + ApplicationHttpClient.GetResponse response = appHttpClient.get(appUrl, token, endpoint); + if (response.getCode() < 200 || response.getCode() >= 300) { + throw new IllegalStateException( + format("Error while executing a call to GitHub. Return code %s. Error message: %s.", response.getCode(), response.getContent().orElse(""))); + } + return response; + } } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImpl.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImpl.java deleted file mode 100644 index 36d5768ca59..00000000000 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImpl.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.alm.client.github; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Nullable; -import org.kohsuke.github.GHRateLimit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.alm.client.github.security.AccessToken; -import org.sonar.api.ce.ComputeEngineSide; -import org.sonar.api.server.ServerSide; - -import static java.lang.String.format; - -@ServerSide -@ComputeEngineSide -public class GithubPaginatedHttpClientImpl implements GithubPaginatedHttpClient { - - private static final Logger LOG = LoggerFactory.getLogger(GithubPaginatedHttpClientImpl.class); - private final GithubApplicationHttpClient appHttpClient; - private final RatioBasedRateLimitChecker rateLimitChecker; - - public GithubPaginatedHttpClientImpl(GithubApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { - this.appHttpClient = appHttpClient; - this.rateLimitChecker = rateLimitChecker; - } - - @Override - public List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException { - List results = new ArrayList<>(); - String nextEndpoint = query + "?per_page=100"; - if (query.contains("?")) { - nextEndpoint = query + "&per_page=100"; - } - GithubApplicationHttpClient.RateLimit rateLimit = null; - while (nextEndpoint != null) { - checkRateLimit(rateLimit); - GithubApplicationHttpClient.GetResponse response = executeCall(appUrl, token, nextEndpoint); - response.getContent() - .ifPresent(content -> results.addAll(responseDeserializer.apply(content))); - nextEndpoint = response.getNextEndPoint().orElse(null); - rateLimit = response.getRateLimit(); - } - return results; - } - - private void checkRateLimit(@Nullable GithubApplicationHttpClient.RateLimit rateLimit) { - if (rateLimit == null) { - return; - } - try { - GHRateLimit.Record rateLimitRecord = new GHRateLimit.Record(rateLimit.limit(), rateLimit.remaining(), rateLimit.reset()); - rateLimitChecker.checkRateLimit(rateLimitRecord, 0); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn(format("Thread interrupted: %s", e.getMessage()), e); - } - } - - private GithubApplicationHttpClient.GetResponse executeCall(String appUrl, AccessToken token, String endpoint) throws IOException { - GithubApplicationHttpClient.GetResponse response = appHttpClient.get(appUrl, token, endpoint); - if (response.getCode() < 200 || response.getCode() >= 300) { - throw new IllegalStateException( - format("Error while executing a call to GitHub. Return code %s. Error message: %s.", response.getCode(), response.getContent().orElse(""))); - } - return response; - } -} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java new file mode 100644 index 00000000000..134e942671e --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/PaginatedHttpClient.java @@ -0,0 +1,30 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import java.io.IOException; +import java.util.List; +import java.util.function.Function; +import org.sonar.alm.client.github.security.AccessToken; + +public interface PaginatedHttpClient { + + List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException; +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java index 7ba266f4740..9eb6e46e493 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java @@ -34,19 +34,19 @@ public class RatioBasedRateLimitChecker extends RateLimitChecker { @VisibleForTesting static final String RATE_RATIO_EXCEEDED_MESSAGE = "The GitHub API rate limit is almost reached. Pausing GitHub provisioning until the next rate limit reset. " - + "{} out of {} calls were used."; + + "{} out of {} calls were used."; private static final int MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING = 90; - @Override - public boolean checkRateLimit(GHRateLimit.Record rateLimitRecord, long count) throws InterruptedException { - int limit = rateLimitRecord.getLimit(); - int apiCallsUsed = limit - rateLimitRecord.getRemaining(); + public boolean checkRateLimit(ApplicationHttpClient.RateLimit rateLimitRecord) throws InterruptedException { + int limit = rateLimitRecord.limit(); + int apiCallsUsed = limit - rateLimitRecord.remaining(); double percentageOfCallsUsed = computePercentageOfCallsUsed(apiCallsUsed, limit); LOGGER.debug("{} GitHub API calls used of {} available per hours", apiCallsUsed, limit); if (percentageOfCallsUsed >= MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING) { LOGGER.warn(RATE_RATIO_EXCEEDED_MESSAGE, apiCallsUsed, limit); - return sleepUntilReset(rateLimitRecord); + GHRateLimit.Record rateLimit = new GHRateLimit.Record(rateLimitRecord.limit(), rateLimitRecord.remaining(), rateLimitRecord.reset()); + return sleepUntilReset(rateLimit); } return false; } diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java index ab1f4b5fb69..b93540c9057 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/gitlab/GitlabHttpClient.java @@ -35,10 +35,10 @@ import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; import org.apache.logging.log4j.util.Strings; -import org.sonar.alm.client.TimeoutConfiguration; -import org.sonar.api.server.ServerSide; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.alm.client.TimeoutConfiguration; +import org.sonar.api.server.ServerSide; import org.sonarqube.ws.MediaTypes; import org.sonarqube.ws.client.OkHttpClientBuilder; @@ -324,6 +324,34 @@ public class GitlabHttpClient { } } + /*public void getGroups(String gitlabUrl, String token) { + String url = String.format("%s/groups", gitlabUrl); + LOG.debug(String.format("get groups : [%s]", url)); + + Request request = new Request.Builder() + .addHeader(PRIVATE_TOKEN, token) + .url(url) + .get() + .build(); + + + try (Response response = client.newCall(request).execute()) { + Headers headers = response.headers(); + checkResponseIsSuccessful(response, "Could not get projects from GitLab instance"); + List projectList = Project.parseJsonArray(response.body().string()); + int returnedPageNumber = parseAndGetIntegerHeader(headers.get("X-Page")); + int returnedPageSize = parseAndGetIntegerHeader(headers.get("X-Per-Page")); + String xtotal = headers.get("X-Total"); + Integer totalProjects = Strings.isEmpty(xtotal) ? null : parseAndGetIntegerHeader(xtotal); + return new ProjectList(projectList, returnedPageNumber, returnedPageSize, totalProjects); + } catch (JsonSyntaxException e) { + throw new IllegalArgumentException("Could not parse GitLab answer to search projects. Got a non-json payload as result."); + } catch (IOException e) { + logException(url, e); + throw new IllegalStateException(e.getMessage(), e); + } + }*/ + private static int parseAndGetIntegerHeader(@Nullable String header) { if (header == null) { throw new IllegalArgumentException("Pagination data from GitLab response is missing"); diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GenericApplicationHttpClientTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GenericApplicationHttpClientTest.java new file mode 100644 index 00000000000..53a6fcf08c9 --- /dev/null +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GenericApplicationHttpClientTest.java @@ -0,0 +1,480 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.concurrent.Callable; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okhttp3.mockwebserver.SocketPolicy; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.event.Level; +import org.sonar.alm.client.ConstantTimeoutConfiguration; +import org.sonar.alm.client.TimeoutConfiguration; +import org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; +import org.sonar.alm.client.github.ApplicationHttpClient.Response; +import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.alm.client.github.security.UserAccessToken; +import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; + +import static java.lang.String.format; +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.fail; +import static org.sonar.alm.client.github.ApplicationHttpClient.RateLimit; + +@RunWith(DataProviderRunner.class) +public class GenericApplicationHttpClientTest { + private static final String GH_API_VERSION_HEADER = "X-GitHub-Api-Version"; + private static final String GH_API_VERSION = "2022-11-28"; + + @Rule + public MockWebServer server = new MockWebServer(); + + @ClassRule + public static LogTester logTester = new LogTester().setLevel(LoggerLevel.WARN); + + private GenericApplicationHttpClient underTest; + + private final AccessToken accessToken = new UserAccessToken(randomAlphabetic(10)); + private final String randomEndPoint = "/" + randomAlphabetic(10); + private final String randomBody = randomAlphabetic(40); + private String appUrl; + + @Before + public void setUp() { + this.appUrl = format("http://%s:%s", server.getHostName(), server.getPort()); + this.underTest = new TestApplicationHttpClient(new GithubHeaders(), new ConstantTimeoutConfiguration(500)); + logTester.clear(); + } + + private class TestApplicationHttpClient extends GenericApplicationHttpClient { + public TestApplicationHttpClient(DevopsPlatformHeaders devopsPlatformHeaders, TimeoutConfiguration timeoutConfiguration) { + super(devopsPlatformHeaders, timeoutConfiguration); + } + } + + @Test + public void get_fails_if_endpoint_does_not_start_with_slash() throws IOException { + assertThatThrownBy(() -> underTest.get(appUrl, accessToken, "api/foo/bar")) + .hasMessage("endpoint must start with '/' or 'http'") + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void get_fails_if_endpoint_does_not_start_with_http() throws IOException { + assertThatThrownBy(() -> underTest.get(appUrl, accessToken, "ttp://api/foo/bar")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("endpoint must start with '/' or 'http'"); + } + + @Test + public void get_fails_if_github_endpoint_is_invalid() throws IOException { + assertThatThrownBy(() -> underTest.get("invalidUrl", accessToken, "/endpoint")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("invalidUrl/endpoint is not a valid url"); + } + + @Test + public void getSilent_no_log_if_code_is_not_200() throws IOException { + server.enqueue(new MockResponse().setResponseCode(403)); + + GetResponse response = underTest.getSilent(appUrl, accessToken, randomEndPoint); + + assertThat(logTester.logs()).isEmpty(); + assertThat(response.getContent()).isEmpty(); + + } + + @Test + public void get_log_if_code_is_not_200() throws IOException { + server.enqueue(new MockResponse().setResponseCode(403)); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(logTester.logs(Level.WARN)).isNotEmpty(); + assertThat(response.getContent()).isEmpty(); + + } + + @Test + public void get_adds_authentication_header_with_Bearer_type_and_Accept_header() throws IOException, InterruptedException { + server.enqueue(new MockResponse()); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response).isNotNull(); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getMethod()).isEqualTo("GET"); + assertThat(recordedRequest.getPath()).isEqualTo(randomEndPoint); + assertThat(recordedRequest.getHeader("Authorization")).isEqualTo("token " + accessToken.getValue()); + assertThat(recordedRequest.getHeader(GH_API_VERSION_HEADER)).isEqualTo(GH_API_VERSION); + } + + @Test + public void get_returns_body_as_response_if_code_is_200() throws IOException { + server.enqueue(new MockResponse().setResponseCode(200).setBody(randomBody)); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response.getContent()).contains(randomBody); + } + + @Test + public void get_timeout() { + server.enqueue(new MockResponse().setSocketPolicy(SocketPolicy.NO_RESPONSE)); + + try { + underTest.get(appUrl, accessToken, randomEndPoint); + fail("Expected timeout"); + } catch (Exception e) { + assertThat(e).isInstanceOf(SocketTimeoutException.class); + } + } + + @Test + @UseDataProvider("someHttpCodesWithContentBut200") + public void get_empty_response_if_code_is_not_200(int code) throws IOException { + server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response.getContent()).contains(randomBody); + } + + @Test + public void get_returns_empty_endPoint_when_no_link_header() throws IOException { + server.enqueue(new MockResponse().setBody(randomBody)); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response.getNextEndPoint()).isEmpty(); + } + + @Test + public void get_returns_empty_endPoint_when_link_header_does_not_have_next_rel() throws IOException { + server.enqueue(new MockResponse().setBody(randomBody) + .setHeader("link", "; rel=\"prev\", " + + "; rel=\"first\"")); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response.getNextEndPoint()).isEmpty(); + } + + @Test + @UseDataProvider("linkHeadersWithNextRel") + public void get_returns_endPoint_when_link_header_has_next_rel(String linkHeader) throws IOException { + server.enqueue(new MockResponse().setBody(randomBody) + .setHeader("link", linkHeader)); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response.getNextEndPoint()).contains("https://api.github.com/installation/repositories?per_page=5&page=2"); + } + + @Test + public void get_returns_endPoint_when_link_header_has_next_rel_different_case() throws IOException { + String linkHeader = "; rel=\"next\""; + server.enqueue(new MockResponse().setBody(randomBody) + .setHeader("Link", linkHeader)); + + GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); + + assertThat(response.getNextEndPoint()).contains("https://api.github.com/installation/repositories?per_page=5&page=2"); + } + + @DataProvider + public static Object[][] linkHeadersWithNextRel() { + String expected = "https://api.github.com/installation/repositories?per_page=5&page=2"; + return new Object[][] { + {"<" + expected + ">; rel=\"next\""}, + {"<" + expected + ">; rel=\"next\", " + + "; rel=\"first\""}, + {"; rel=\"first\", " + + "<" + expected + ">; rel=\"next\""}, + {"; rel=\"first\", " + + "<" + expected + ">; rel=\"next\", " + + "; rel=\"last\""}, + }; + } + + @DataProvider + public static Object[][] someHttpCodesWithContentBut200() { + return new Object[][] { + {201}, + {202}, + {203}, + {404}, + {500} + }; + } + + @Test + public void post_fails_if_endpoint_does_not_start_with_slash() throws IOException { + assertThatThrownBy(() -> underTest.post(appUrl, accessToken, "api/foo/bar")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("endpoint must start with '/' or 'http'"); + } + + @Test + public void post_fails_if_endpoint_does_not_start_with_http() throws IOException { + assertThatThrownBy(() -> underTest.post(appUrl, accessToken, "ttp://api/foo/bar")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("endpoint must start with '/' or 'http'"); + } + + @Test + public void post_fails_if_github_endpoint_is_invalid() throws IOException { + assertThatThrownBy(() -> underTest.post("invalidUrl", accessToken, "/endpoint")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("invalidUrl/endpoint is not a valid url"); + } + + @Test + public void post_adds_authentication_header_with_Bearer_type_and_Accept_header() throws IOException, InterruptedException { + server.enqueue(new MockResponse()); + + Response response = underTest.post(appUrl, accessToken, randomEndPoint); + + assertThat(response).isNotNull(); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getMethod()).isEqualTo("POST"); + assertThat(recordedRequest.getPath()).isEqualTo(randomEndPoint); + assertThat(recordedRequest.getHeader("Authorization")).isEqualTo("token " + accessToken.getValue()); + assertThat(recordedRequest.getHeader(GH_API_VERSION_HEADER)).isEqualTo(GH_API_VERSION); + } + + @Test + @DataProvider({"200", "201", "202"}) + public void post_returns_body_as_response_if_success(int code) throws IOException { + server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); + + Response response = underTest.post(appUrl, accessToken, randomEndPoint); + + assertThat(response.getContent()).contains(randomBody); + } + + @Test + public void post_returns_empty_response_if_code_is_204() throws IOException { + server.enqueue(new MockResponse().setResponseCode(204)); + + Response response = underTest.post(appUrl, accessToken, randomEndPoint); + + assertThat(response.getContent()).isEmpty(); + } + + @Test + @UseDataProvider("httpCodesBut200_201And204") + public void post_has_json_error_in_body_if_code_is_neither_200_201_nor_204(int code) throws IOException { + server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); + + Response response = underTest.post(appUrl, accessToken, randomEndPoint); + + assertThat(response.getContent()).contains(randomBody); + } + + @DataProvider + public static Object[][] httpCodesBut200_201And204() { + return new Object[][] { + {202}, + {203}, + {400}, + {401}, + {403}, + {404}, + {500} + }; + } + + @Test + public void post_with_json_body_adds_json_to_body_request() throws IOException, InterruptedException { + server.enqueue(new MockResponse()); + String jsonBody = "{\"foo\": \"bar\"}"; + Response response = underTest.post(appUrl, accessToken, randomEndPoint, jsonBody); + + assertThat(response).isNotNull(); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getBody().readUtf8()).isEqualTo(jsonBody); + } + + @Test + public void patch_with_json_body_adds_json_to_body_request() throws IOException, InterruptedException { + server.enqueue(new MockResponse()); + String jsonBody = "{\"foo\": \"bar\"}"; + + Response response = underTest.patch(appUrl, accessToken, randomEndPoint, jsonBody); + + assertThat(response).isNotNull(); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getBody().readUtf8()).isEqualTo(jsonBody); + } + + @Test + public void patch_returns_body_as_response_if_code_is_200() throws IOException { + server.enqueue(new MockResponse().setResponseCode(200).setBody(randomBody)); + + Response response = underTest.patch(appUrl, accessToken, randomEndPoint, "{}"); + + assertThat(response.getContent()).contains(randomBody); + } + + @Test + public void patch_returns_empty_response_if_code_is_204() throws IOException { + server.enqueue(new MockResponse().setResponseCode(204)); + + Response response = underTest.patch(appUrl, accessToken, randomEndPoint, "{}"); + + assertThat(response.getContent()).isEmpty(); + } + + @Test + public void delete_returns_empty_response_if_code_is_204() throws IOException { + server.enqueue(new MockResponse().setResponseCode(204)); + + Response response = underTest.delete(appUrl, accessToken, randomEndPoint); + + assertThat(response.getContent()).isEmpty(); + } + + @DataProvider + public static Object[][] httpCodesBut204() { + return new Object[][] { + {200}, + {201}, + {202}, + {203}, + {400}, + {401}, + {403}, + {404}, + {500} + }; + } + + @Test + @UseDataProvider("httpCodesBut204") + public void delete_returns_response_if_code_is_not_204(int code) throws IOException { + server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); + + Response response = underTest.delete(appUrl, accessToken, randomEndPoint); + + assertThat(response.getContent()).hasValue(randomBody); + } + + @DataProvider + public static Object[][] httpCodesBut200And204() { + return new Object[][] { + {201}, + {202}, + {203}, + {400}, + {401}, + {403}, + {404}, + {500} + }; + } + + @Test + @UseDataProvider("httpCodesBut200And204") + public void patch_has_json_error_in_body_if_code_is_neither_200_nor_204(int code) throws IOException { + server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); + + Response response = underTest.patch(appUrl, accessToken, randomEndPoint, "{}"); + + assertThat(response.getContent()).contains(randomBody); + } + + @Test + public void get_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { + testRateLimitHeader(() -> underTest.get(appUrl, accessToken, randomEndPoint)); + } + + private void testRateLimitHeader(Callable request ) throws Exception { + server.enqueue(new MockResponse().setBody(randomBody) + .setHeader("x-ratelimit-remaining", "1") + .setHeader("x-ratelimit-limit", "10") + .setHeader("x-ratelimit-reset", "1000")); + + Response response = request.call(); + + assertThat(response.getRateLimit()) + .isEqualTo(new RateLimit(1, 10, 1000L)); + } + + @Test + public void get_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { + + testMissingRateLimitHeader(() -> underTest.get(appUrl, accessToken, randomEndPoint)); + + } + + private void testMissingRateLimitHeader(Callable request ) throws Exception { + server.enqueue(new MockResponse().setBody(randomBody)); + + Response response = request.call(); + assertThat(response.getRateLimit()) + .isNull(); + } + + @Test + public void delete_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { + testRateLimitHeader(() -> underTest.delete(appUrl, accessToken, randomEndPoint)); + + } + + @Test + public void delete_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { + testMissingRateLimitHeader(() -> underTest.delete(appUrl, accessToken, randomEndPoint)); + + } + + @Test + public void patch_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { + testRateLimitHeader(() -> underTest.patch(appUrl, accessToken, randomEndPoint, "body")); + } + + @Test + public void patch_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { + testMissingRateLimitHeader(() -> underTest.patch(appUrl, accessToken, randomEndPoint, "body")); + } + + @Test + public void post_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { + testRateLimitHeader(() -> underTest.post(appUrl, accessToken, randomEndPoint)); + } + + @Test + public void post_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { + testMissingRateLimitHeader(() -> underTest.post(appUrl, accessToken, randomEndPoint)); + } +} diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java index cbe044b9bfe..6cf2f71dfc9 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java @@ -36,7 +36,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.slf4j.event.Level; -import org.sonar.alm.client.github.GithubApplicationHttpClient.RateLimit; +import org.sonar.alm.client.github.ApplicationHttpClient.RateLimit; import org.sonar.alm.client.github.api.GsonRepositoryCollaborator; import org.sonar.alm.client.github.api.GsonRepositoryTeam; import org.sonar.alm.client.github.config.GithubAppConfiguration; @@ -68,7 +68,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.sonar.alm.client.github.GithubApplicationHttpClient.GetResponse; +import static org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; @RunWith(DataProviderRunner.class) public class GithubApplicationClientImplTest { @@ -114,12 +114,12 @@ public class GithubApplicationClientImplTest { @ClassRule public static LogTester logTester = new LogTester().setLevel(LoggerLevel.WARN); - private GithubApplicationHttpClientImpl httpClient = mock(); + private GenericApplicationHttpClient httpClient = mock(); private GithubAppSecurity appSecurity = mock(); private GithubAppConfiguration githubAppConfiguration = mock(); private GitHubSettings gitHubSettings = mock(); - private GithubPaginatedHttpClient githubPaginatedHttpClient = mock(); + private PaginatedHttpClient githubPaginatedHttpClient = mock(); private AppInstallationToken appInstallationToken = mock(); private GithubApplicationClient underTest; diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationHttpClientImplTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationHttpClientImplTest.java deleted file mode 100644 index 55e6a4cc207..00000000000 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationHttpClientImplTest.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.alm.client.github; - -import com.tngtech.java.junit.dataprovider.DataProvider; -import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import com.tngtech.java.junit.dataprovider.UseDataProvider; -import java.io.IOException; -import java.net.SocketTimeoutException; -import java.util.concurrent.Callable; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import okhttp3.mockwebserver.SocketPolicy; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.slf4j.event.Level; -import org.sonar.alm.client.ConstantTimeoutConfiguration; -import org.sonar.alm.client.github.GithubApplicationHttpClient.GetResponse; -import org.sonar.alm.client.github.GithubApplicationHttpClient.Response; -import org.sonar.alm.client.github.security.AccessToken; -import org.sonar.alm.client.github.security.UserAccessToken; -import org.sonar.api.testfixtures.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; - -import static java.lang.String.format; -import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.Assert.fail; -import static org.sonar.alm.client.github.GithubApplicationHttpClient.RateLimit; - -@RunWith(DataProviderRunner.class) -public class GithubApplicationHttpClientImplTest { - private static final String GH_API_VERSION_HEADER = "X-GitHub-Api-Version"; - private static final String GH_API_VERSION = "2022-11-28"; - - @Rule - public MockWebServer server = new MockWebServer(); - - @ClassRule - public static LogTester logTester = new LogTester().setLevel(LoggerLevel.WARN); - - private GithubApplicationHttpClientImpl underTest; - - private final AccessToken accessToken = new UserAccessToken(randomAlphabetic(10)); - private final String randomEndPoint = "/" + randomAlphabetic(10); - private final String randomBody = randomAlphabetic(40); - private String appUrl; - - @Before - public void setUp() { - this.appUrl = format("http://%s:%s", server.getHostName(), server.getPort()); - this.underTest = new GithubApplicationHttpClientImpl(new ConstantTimeoutConfiguration(500)); - logTester.clear(); - } - - @Test - public void get_fails_if_endpoint_does_not_start_with_slash() throws IOException { - assertThatThrownBy(() -> underTest.get(appUrl, accessToken, "api/foo/bar")) - .hasMessage("endpoint must start with '/' or 'http'") - .isInstanceOf(IllegalArgumentException.class); - } - - @Test - public void get_fails_if_endpoint_does_not_start_with_http() throws IOException { - assertThatThrownBy(() -> underTest.get(appUrl, accessToken, "ttp://api/foo/bar")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("endpoint must start with '/' or 'http'"); - } - - @Test - public void get_fails_if_github_endpoint_is_invalid() throws IOException { - assertThatThrownBy(() -> underTest.get("invalidUrl", accessToken, "/endpoint")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("invalidUrl/endpoint is not a valid url"); - } - - @Test - public void getSilent_no_log_if_code_is_not_200() throws IOException { - server.enqueue(new MockResponse().setResponseCode(403)); - - GetResponse response = underTest.getSilent(appUrl, accessToken, randomEndPoint); - - assertThat(logTester.logs()).isEmpty(); - assertThat(response.getContent()).isEmpty(); - - } - - @Test - public void get_log_if_code_is_not_200() throws IOException { - server.enqueue(new MockResponse().setResponseCode(403)); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(logTester.logs(Level.WARN)).isNotEmpty(); - assertThat(response.getContent()).isEmpty(); - - } - - @Test - public void get_adds_authentication_header_with_Bearer_type_and_Accept_header() throws IOException, InterruptedException { - server.enqueue(new MockResponse()); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(response).isNotNull(); - RecordedRequest recordedRequest = server.takeRequest(); - assertThat(recordedRequest.getMethod()).isEqualTo("GET"); - assertThat(recordedRequest.getPath()).isEqualTo(randomEndPoint); - assertThat(recordedRequest.getHeader("Authorization")).isEqualTo("token " + accessToken.getValue()); - assertThat(recordedRequest.getHeader(GH_API_VERSION_HEADER)).isEqualTo(GH_API_VERSION); - } - - @Test - public void get_returns_body_as_response_if_code_is_200() throws IOException { - server.enqueue(new MockResponse().setResponseCode(200).setBody(randomBody)); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(response.getContent()).contains(randomBody); - } - - @Test - public void get_timeout() { - server.enqueue(new MockResponse().setSocketPolicy(SocketPolicy.NO_RESPONSE)); - - try { - underTest.get(appUrl, accessToken, randomEndPoint); - fail("Expected timeout"); - } catch (Exception e) { - assertThat(e).isInstanceOf(SocketTimeoutException.class); - } - } - - @Test - @UseDataProvider("someHttpCodesWithContentBut200") - public void get_empty_response_if_code_is_not_200(int code) throws IOException { - server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(response.getContent()).contains(randomBody); - } - - @Test - public void get_returns_empty_endPoint_when_no_link_header() throws IOException { - server.enqueue(new MockResponse().setBody(randomBody)); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(response.getNextEndPoint()).isEmpty(); - } - - @Test - public void get_returns_empty_endPoint_when_link_header_does_not_have_next_rel() throws IOException { - server.enqueue(new MockResponse().setBody(randomBody) - .setHeader("link", "; rel=\"prev\", " + - "; rel=\"first\"")); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(response.getNextEndPoint()).isEmpty(); - } - - @Test - @UseDataProvider("linkHeadersWithNextRel") - public void get_returns_endPoint_when_link_header_has_next_rel(String linkHeader) throws IOException { - server.enqueue(new MockResponse().setBody(randomBody) - .setHeader("link", linkHeader)); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(response.getNextEndPoint()).contains("https://api.github.com/installation/repositories?per_page=5&page=2"); - } - - @Test - public void get_returns_endPoint_when_link_header_has_next_rel_different_case() throws IOException { - String linkHeader = "; rel=\"next\""; - server.enqueue(new MockResponse().setBody(randomBody) - .setHeader("Link", linkHeader)); - - GetResponse response = underTest.get(appUrl, accessToken, randomEndPoint); - - assertThat(response.getNextEndPoint()).contains("https://api.github.com/installation/repositories?per_page=5&page=2"); - } - - @DataProvider - public static Object[][] linkHeadersWithNextRel() { - String expected = "https://api.github.com/installation/repositories?per_page=5&page=2"; - return new Object[][] { - {"<" + expected + ">; rel=\"next\""}, - {"<" + expected + ">; rel=\"next\", " + - "; rel=\"first\""}, - {"; rel=\"first\", " + - "<" + expected + ">; rel=\"next\""}, - {"; rel=\"first\", " + - "<" + expected + ">; rel=\"next\", " + - "; rel=\"last\""}, - }; - } - - @DataProvider - public static Object[][] someHttpCodesWithContentBut200() { - return new Object[][] { - {201}, - {202}, - {203}, - {404}, - {500} - }; - } - - @Test - public void post_fails_if_endpoint_does_not_start_with_slash() throws IOException { - assertThatThrownBy(() -> underTest.post(appUrl, accessToken, "api/foo/bar")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("endpoint must start with '/' or 'http'"); - } - - @Test - public void post_fails_if_endpoint_does_not_start_with_http() throws IOException { - assertThatThrownBy(() -> underTest.post(appUrl, accessToken, "ttp://api/foo/bar")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("endpoint must start with '/' or 'http'"); - } - - @Test - public void post_fails_if_github_endpoint_is_invalid() throws IOException { - assertThatThrownBy(() -> underTest.post("invalidUrl", accessToken, "/endpoint")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("invalidUrl/endpoint is not a valid url"); - } - - @Test - public void post_adds_authentication_header_with_Bearer_type_and_Accept_header() throws IOException, InterruptedException { - server.enqueue(new MockResponse()); - - Response response = underTest.post(appUrl, accessToken, randomEndPoint); - - assertThat(response).isNotNull(); - RecordedRequest recordedRequest = server.takeRequest(); - assertThat(recordedRequest.getMethod()).isEqualTo("POST"); - assertThat(recordedRequest.getPath()).isEqualTo(randomEndPoint); - assertThat(recordedRequest.getHeader("Authorization")).isEqualTo("token " + accessToken.getValue()); - assertThat(recordedRequest.getHeader(GH_API_VERSION_HEADER)).isEqualTo(GH_API_VERSION); - } - - @Test - @DataProvider({"200", "201", "202"}) - public void post_returns_body_as_response_if_success(int code) throws IOException { - server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); - - Response response = underTest.post(appUrl, accessToken, randomEndPoint); - - assertThat(response.getContent()).contains(randomBody); - } - - @Test - public void post_returns_empty_response_if_code_is_204() throws IOException { - server.enqueue(new MockResponse().setResponseCode(204)); - - Response response = underTest.post(appUrl, accessToken, randomEndPoint); - - assertThat(response.getContent()).isEmpty(); - } - - @Test - @UseDataProvider("httpCodesBut200_201And204") - public void post_has_json_error_in_body_if_code_is_neither_200_201_nor_204(int code) throws IOException { - server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); - - Response response = underTest.post(appUrl, accessToken, randomEndPoint); - - assertThat(response.getContent()).contains(randomBody); - } - - @DataProvider - public static Object[][] httpCodesBut200_201And204() { - return new Object[][] { - {202}, - {203}, - {400}, - {401}, - {403}, - {404}, - {500} - }; - } - - @Test - public void post_with_json_body_adds_json_to_body_request() throws IOException, InterruptedException { - server.enqueue(new MockResponse()); - String jsonBody = "{\"foo\": \"bar\"}"; - Response response = underTest.post(appUrl, accessToken, randomEndPoint, jsonBody); - - assertThat(response).isNotNull(); - RecordedRequest recordedRequest = server.takeRequest(); - assertThat(recordedRequest.getBody().readUtf8()).isEqualTo(jsonBody); - } - - @Test - public void patch_with_json_body_adds_json_to_body_request() throws IOException, InterruptedException { - server.enqueue(new MockResponse()); - String jsonBody = "{\"foo\": \"bar\"}"; - - Response response = underTest.patch(appUrl, accessToken, randomEndPoint, jsonBody); - - assertThat(response).isNotNull(); - RecordedRequest recordedRequest = server.takeRequest(); - assertThat(recordedRequest.getBody().readUtf8()).isEqualTo(jsonBody); - } - - @Test - public void patch_returns_body_as_response_if_code_is_200() throws IOException { - server.enqueue(new MockResponse().setResponseCode(200).setBody(randomBody)); - - Response response = underTest.patch(appUrl, accessToken, randomEndPoint, "{}"); - - assertThat(response.getContent()).contains(randomBody); - } - - @Test - public void patch_returns_empty_response_if_code_is_204() throws IOException { - server.enqueue(new MockResponse().setResponseCode(204)); - - Response response = underTest.patch(appUrl, accessToken, randomEndPoint, "{}"); - - assertThat(response.getContent()).isEmpty(); - } - - @Test - public void delete_returns_empty_response_if_code_is_204() throws IOException { - server.enqueue(new MockResponse().setResponseCode(204)); - - Response response = underTest.delete(appUrl, accessToken, randomEndPoint); - - assertThat(response.getContent()).isEmpty(); - } - - @DataProvider - public static Object[][] httpCodesBut204() { - return new Object[][] { - {200}, - {201}, - {202}, - {203}, - {400}, - {401}, - {403}, - {404}, - {500} - }; - } - - @Test - @UseDataProvider("httpCodesBut204") - public void delete_returns_response_if_code_is_not_204(int code) throws IOException { - server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); - - Response response = underTest.delete(appUrl, accessToken, randomEndPoint); - - assertThat(response.getContent()).hasValue(randomBody); - } - - @DataProvider - public static Object[][] httpCodesBut200And204() { - return new Object[][] { - {201}, - {202}, - {203}, - {400}, - {401}, - {403}, - {404}, - {500} - }; - } - - @Test - @UseDataProvider("httpCodesBut200And204") - public void patch_has_json_error_in_body_if_code_is_neither_200_nor_204(int code) throws IOException { - server.enqueue(new MockResponse().setResponseCode(code).setBody(randomBody)); - - Response response = underTest.patch(appUrl, accessToken, randomEndPoint, "{}"); - - assertThat(response.getContent()).contains(randomBody); - } - - @Test - public void get_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.get(appUrl, accessToken, randomEndPoint)); - } - - private void testRateLimitHeader(Callable request ) throws Exception { - server.enqueue(new MockResponse().setBody(randomBody) - .setHeader("x-ratelimit-remaining", "1") - .setHeader("x-ratelimit-limit", "10") - .setHeader("x-ratelimit-reset", "1000")); - - Response response = request.call(); - - assertThat(response.getRateLimit()) - .isEqualTo(new RateLimit(1, 10, 1000L)); - } - - @Test - public void get_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { - - testMissingRateLimitHeader(() -> underTest.get(appUrl, accessToken, randomEndPoint)); - - } - - private void testMissingRateLimitHeader(Callable request ) throws Exception { - server.enqueue(new MockResponse().setBody(randomBody)); - - Response response = request.call(); - assertThat(response.getRateLimit()) - .isNull(); - } - - @Test - public void delete_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.delete(appUrl, accessToken, randomEndPoint)); - - } - - @Test - public void delete_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { - testMissingRateLimitHeader(() -> underTest.delete(appUrl, accessToken, randomEndPoint)); - - } - - @Test - public void patch_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.patch(appUrl, accessToken, randomEndPoint, "body")); - } - - @Test - public void patch_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { - testMissingRateLimitHeader(() -> underTest.patch(appUrl, accessToken, randomEndPoint, "body")); - } - - @Test - public void post_whenRateLimitHeadersArePresent_returnsRateLimit() throws Exception { - testRateLimitHeader(() -> underTest.post(appUrl, accessToken, randomEndPoint)); - } - - @Test - public void post_whenRateLimitHeadersAreMissing_returnsNull() throws Exception { - testMissingRateLimitHeader(() -> underTest.post(appUrl, accessToken, randomEndPoint)); - } -} diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java index bc11a17e531..5df514c857e 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java @@ -28,7 +28,6 @@ import java.util.Optional; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; -import org.kohsuke.github.GHRateLimit; import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; @@ -41,13 +40,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalStateException; import static org.assertj.core.api.Assertions.assertThatNoException; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.sonar.alm.client.github.GithubApplicationHttpClient.GetResponse; +import static org.sonar.alm.client.github.ApplicationHttpClient.GetResponse; @RunWith(MockitoJUnitRunner.class) public class GithubPaginatedHttpClientImplTest { @@ -70,10 +68,10 @@ public class GithubPaginatedHttpClientImplTest { RatioBasedRateLimitChecker rateLimitChecker; @Mock - GithubApplicationHttpClient appHttpClient; + ApplicationHttpClient appHttpClient; @InjectMocks - private GithubPaginatedHttpClientImpl underTest; + private GithubPaginatedHttpClient underTest; @Test public void get_whenNoPagination_ReturnsCorrectResponse() throws IOException { @@ -118,19 +116,19 @@ public class GithubPaginatedHttpClientImplTest { assertThat(results) .containsExactly("result1", "result2", "result3"); - ArgumentCaptor rateLimitRecordCaptor = ArgumentCaptor.forClass(GHRateLimit.Record.class); - verify(rateLimitChecker).checkRateLimit(rateLimitRecordCaptor.capture(), eq(0L)); - GHRateLimit.Record rateLimitRecord = rateLimitRecordCaptor.getValue(); - assertThat(rateLimitRecord.getLimit()).isEqualTo(10); - assertThat(rateLimitRecord.getRemaining()).isEqualTo(1); - assertThat(rateLimitRecord.getResetEpochSeconds()).isZero(); + ArgumentCaptor rateLimitRecordCaptor = ArgumentCaptor.forClass(ApplicationHttpClient.RateLimit.class); + verify(rateLimitChecker).checkRateLimit(rateLimitRecordCaptor.capture()); + ApplicationHttpClient.RateLimit rateLimitRecord = rateLimitRecordCaptor.getValue(); + assertThat(rateLimitRecord.limit()).isEqualTo(10); + assertThat(rateLimitRecord.remaining()).isEqualTo(1); + assertThat(rateLimitRecord.reset()).isZero(); } private static GetResponse mockResponseWithPaginationAndRateLimit(String content, String nextEndpoint) { GetResponse response = mockResponseWithoutPagination(content); when(response.getCode()).thenReturn(200); when(response.getNextEndPoint()).thenReturn(Optional.of(nextEndpoint)); - when(response.getRateLimit()).thenReturn(new GithubApplicationHttpClient.RateLimit(1, 10, 0L)); + when(response.getRateLimit()).thenReturn(new ApplicationHttpClient.RateLimit(1, 10, 0L)); return response; } @@ -159,7 +157,7 @@ public class GithubPaginatedHttpClientImplTest { GetResponse response2 = mockResponseWithoutPagination("[\"result3\"]"); when(appHttpClient.get(APP_URL, accessToken, ENDPOINT + "?per_page=100")).thenReturn(response1); when(appHttpClient.get(APP_URL, accessToken, "/next-endpoint")).thenReturn(response2); - doThrow(new InterruptedException("interrupted")).when(rateLimitChecker).checkRateLimit(any(GHRateLimit.Record.class), anyLong()); + doThrow(new InterruptedException("interrupted")).when(rateLimitChecker).checkRateLimit(any(ApplicationHttpClient.RateLimit.class)); assertThatNoException() .isThrownBy(() -> underTest.get(APP_URL, accessToken, ENDPOINT, result -> gson.fromJson(result, STRING_LIST_TYPE))); diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java index 83913b19cb4..d10633365c6 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java @@ -22,17 +22,13 @@ package org.sonar.alm.client.github; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; -import java.sql.Date; -import java.time.temporal.ChronoUnit; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; -import org.kohsuke.github.GHRateLimit; import org.slf4j.event.Level; import org.sonar.api.testfixtures.log.LogTester; import static java.lang.String.format; -import static java.time.Instant.now; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -63,19 +59,19 @@ public class RatioBasedRateLimitCheckerTest { @Test @UseDataProvider("rates") public void checkRateLimit(int limit, int remaining, boolean rateLimitShouldBeExceeded) throws InterruptedException { - GHRateLimit.Record record = mock(); - when(record.getLimit()).thenReturn(limit); - when(record.getRemaining()).thenReturn(remaining); - when(record.getResetDate()).thenReturn(Date.from(now().plus(100, ChronoUnit.MILLIS))); + ApplicationHttpClient.RateLimit record = mock(); + when(record.limit()).thenReturn(limit); + when(record.remaining()).thenReturn(remaining); + when(record.reset()).thenReturn(System.currentTimeMillis() / 1000 + 1); long start = System.currentTimeMillis(); - boolean result = ratioBasedRateLimitChecker.checkRateLimit(record, 10); + boolean result = ratioBasedRateLimitChecker.checkRateLimit(record); long stop = System.currentTimeMillis(); long totalTime = stop - start; if (rateLimitShouldBeExceeded) { assertThat(result).isTrue(); - assertThat(stop).isGreaterThanOrEqualTo(record.getResetDate().getTime()); + assertThat(stop).isGreaterThanOrEqualTo(record.reset()); assertThat(logTester.logs(Level.WARN)).contains( format(RATE_RATIO_EXCEEDED_MESSAGE.replaceAll("\\{\\}", "%s"), limit - remaining, limit)); } else { diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 5798a9dadbb..fc9b920c860 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -28,9 +28,10 @@ import org.sonar.alm.client.bitbucket.bitbucketcloud.BitbucketCloudValidator; import org.sonar.alm.client.bitbucketserver.BitbucketServerRestClient; import org.sonar.alm.client.bitbucketserver.BitbucketServerSettingsValidator; import org.sonar.alm.client.github.GithubApplicationClientImpl; -import org.sonar.alm.client.github.GithubApplicationHttpClientImpl; +import org.sonar.alm.client.github.GithubApplicationHttpClient; import org.sonar.alm.client.github.GithubGlobalSettingsValidator; -import org.sonar.alm.client.github.GithubPaginatedHttpClientImpl; +import org.sonar.alm.client.github.GithubHeaders; +import org.sonar.alm.client.github.GithubPaginatedHttpClient; import org.sonar.alm.client.github.GithubPermissionConverter; import org.sonar.alm.client.github.RatioBasedRateLimitChecker; import org.sonar.alm.client.github.config.GithubProvisioningConfigValidator; @@ -555,8 +556,9 @@ public class PlatformLevel4 extends PlatformLevel { RatioBasedRateLimitChecker.class, GithubAppSecurityImpl.class, GithubApplicationClientImpl.class, - GithubPaginatedHttpClientImpl.class, - GithubApplicationHttpClientImpl.class, + GithubPaginatedHttpClient.class, + GithubHeaders.class, + GithubApplicationHttpClient.class, GithubProvisioningConfigValidator.class, GithubProvisioningWs.class, GithubProjectCreatorFactory.class,