From: Aurelien Poscia Date: Wed, 9 Aug 2023 11:26:00 +0000 (+0200) Subject: SONAR-19945 allow GitHub provisioning on more than 30 orgs X-Git-Tag: 10.2.0.77647~247 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=2f2b64a42548a18b6fbbbb6549f98555424f5e5c;p=sonarqube.git SONAR-19945 allow GitHub provisioning on more than 30 orgs --- diff --git a/build.gradle b/build.gradle index 9fb58f044ee..869b36ed3a8 100644 --- a/build.gradle +++ b/build.gradle @@ -415,6 +415,7 @@ subprojects { dependency 'org.hibernate:hibernate-validator:6.2.5.Final' dependency 'javax.el:javax.el-api:3.0.0' dependency 'org.glassfish:javax.el:3.0.0' + dependency 'org.kohsuke:github-api:1.315' // please keep this list alphabetically ordered } diff --git a/server/sonar-alm-client/build.gradle b/server/sonar-alm-client/build.gradle index aa6ab63d148..caafa3f2a41 100644 --- a/server/sonar-alm-client/build.gradle +++ b/server/sonar-alm-client/build.gradle @@ -8,6 +8,7 @@ dependencies { api 'com.google.guava:guava' api 'com.squareup.okhttp3:okhttp' api 'commons-codec:commons-codec' + api 'org.kohsuke:github-api' api 'com.auth0:java-jwt' api 'org.bouncycastle:bcpkix-jdk18on:1.74' api 'org.sonarsource.api.plugin:sonar-plugin-api' diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java index c1f9e28a8c8..ceb5a2f93ab 100644 --- a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubApplicationClientImpl.java @@ -20,7 +20,9 @@ package org.sonar.alm.client.github; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import java.io.IOException; +import java.lang.reflect.Type; import java.net.URI; import java.util.Arrays; import java.util.HashMap; @@ -32,6 +34,8 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.alm.client.github.GithubApplicationHttpClient.GetResponse; import org.sonar.alm.client.github.GithubBinding.GsonGithubRepository; import org.sonar.alm.client.github.GithubBinding.GsonInstallations; @@ -44,8 +48,6 @@ import org.sonar.alm.client.github.security.GithubAppSecurity; import org.sonar.alm.client.github.security.UserAccessToken; import org.sonar.alm.client.gitlab.GsonApp; import org.sonar.api.internal.apachecommons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.auth.github.GitHubSettings; import org.sonar.server.exceptions.ServerException; import org.sonarqube.ws.client.HttpException; @@ -64,15 +66,17 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { protected static final String WRITE_PERMISSION_NAME = "write"; protected static final String READ_PERMISSION_NAME = "read"; protected static final String FAILED_TO_REQUEST_BEGIN_MSG = "Failed to request "; - + private static final Type ORGANIZATION_LIST_TYPE = TypeToken.getParameterized(List.class, GithubBinding.GsonInstallation.class).getType(); protected final GithubApplicationHttpClient appHttpClient; protected final GithubAppSecurity appSecurity; private final GitHubSettings gitHubSettings; - - public GithubApplicationClientImpl(GithubApplicationHttpClient appHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings) { + private final GithubPaginatedHttpClient githubPaginatedHttpClient; + public GithubApplicationClientImpl(GithubApplicationHttpClient appHttpClient, GithubAppSecurity appSecurity, GitHubSettings gitHubSettings, + GithubPaginatedHttpClient githubPaginatedHttpClient) { this.appHttpClient = appHttpClient; this.appSecurity = appSecurity; this.gitHubSettings = gitHubSettings; + this.githubPaginatedHttpClient = githubPaginatedHttpClient; } private static void checkPageArgs(int page, int pageSize) { @@ -170,14 +174,14 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { @Override public List getWhitelistedGithubAppInstallations(GithubAppConfiguration githubAppConfiguration) { - GithubBinding.GsonInstallation[] gsonAppInstallations = fetchAppInstallationsFromGithub(githubAppConfiguration); + List gsonAppInstallations = fetchAppInstallationsFromGithub(githubAppConfiguration); Set allowedOrganizations = gitHubSettings.getOrganizations(); return convertToGithubAppInstallationAndFilterWhitelisted(gsonAppInstallations, allowedOrganizations); } - private static List convertToGithubAppInstallationAndFilterWhitelisted(GithubBinding.GsonInstallation[] gsonAppInstallations, + private static List convertToGithubAppInstallationAndFilterWhitelisted(List gsonAppInstallations, Set allowedOrganizations) { - return Arrays.stream(gsonAppInstallations) + return gsonAppInstallations.stream() .filter(appInstallation -> appInstallation.getAccount().getType().equalsIgnoreCase("Organization")) .map(GithubApplicationClientImpl::toGithubAppInstallation) .filter(appInstallation -> isOrganizationWhiteListed(allowedOrganizations, appInstallation.organizationName())) @@ -196,13 +200,16 @@ public class GithubApplicationClientImpl implements GithubApplicationClient { return allowedOrganizations.isEmpty() || allowedOrganizations.contains(organizationName); } - private GithubBinding.GsonInstallation[] fetchAppInstallationsFromGithub(GithubAppConfiguration githubAppConfiguration) { + private List fetchAppInstallationsFromGithub(GithubAppConfiguration githubAppConfiguration) { AppToken appToken = appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey()); String endpoint = "/app/installations"; - return get(githubAppConfiguration.getApiEndpoint(), appToken, endpoint, - GithubBinding.GsonInstallation[].class).orElseThrow( - () -> new IllegalStateException("An error occurred when retrieving your GitHup App installations. " - + "It might be related to your GitHub App configuration or a connectivity problem.")); + try { + return githubPaginatedHttpClient.get(githubAppConfiguration.getApiEndpoint(), appToken, endpoint, resp -> GSON.fromJson(resp, ORGANIZATION_LIST_TYPE)); + } catch (IOException e) { + LOG.warn(FAILED_TO_REQUEST_BEGIN_MSG + endpoint, e); + throw new IllegalStateException("An error occurred when retrieving your GitHup App installations. " + + "It might be related to your GitHub App configuration or a connectivity problem."); + } } protected Optional get(String baseUrl, AccessToken token, String endPoint, Class gsonClass) { diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java new file mode 100644 index 00000000000..ba4f6379696 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClient.java @@ -0,0 +1,30 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import java.io.IOException; +import java.util.List; +import java.util.function.Function; +import org.sonar.alm.client.github.security.AccessToken; + +public interface GithubPaginatedHttpClient { + + List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException; +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImpl.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImpl.java new file mode 100644 index 00000000000..d2391d0420d --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImpl.java @@ -0,0 +1,86 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Nullable; +import org.kohsuke.github.GHRateLimit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +import static java.lang.String.format; + +@ServerSide +@ComputeEngineSide +public class GithubPaginatedHttpClientImpl implements GithubPaginatedHttpClient { + + private static final Logger LOG = LoggerFactory.getLogger(GithubPaginatedHttpClientImpl.class); + private final GithubApplicationHttpClient appHttpClient; + private final RatioBasedRateLimitChecker rateLimitChecker; + + public GithubPaginatedHttpClientImpl(GithubApplicationHttpClient appHttpClient, RatioBasedRateLimitChecker rateLimitChecker) { + this.appHttpClient = appHttpClient; + this.rateLimitChecker = rateLimitChecker; + } + + @Override + public List get(String appUrl, AccessToken token, String query, Function> responseDeserializer) throws IOException { + List results = new ArrayList<>(); + String nextEndpoint = query + "?per_page=100"; + GithubApplicationHttpClient.RateLimit rateLimit = null; + while (nextEndpoint != null) { + checkRateLimit(rateLimit); + GithubApplicationHttpClient.GetResponse response = executeCall(appUrl, token, nextEndpoint); + response.getContent() + .ifPresent(content -> results.addAll(responseDeserializer.apply(content))); + nextEndpoint = response.getNextEndPoint().orElse(null); + rateLimit = response.getRateLimit(); + } + return results; + } + + private void checkRateLimit(@Nullable GithubApplicationHttpClient.RateLimit rateLimit) { + if (rateLimit == null) { + return; + } + try { + GHRateLimit.Record rateLimitRecord = new GHRateLimit.Record(rateLimit.limit(), rateLimit.remaining(), rateLimit.reset()); + rateLimitChecker.checkRateLimit(rateLimitRecord, 0); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn(format("Thread interrupted: %s", e.getMessage()), e); + } + } + + private GithubApplicationHttpClient.GetResponse executeCall(String appUrl, AccessToken token, String endpoint) throws IOException { + GithubApplicationHttpClient.GetResponse response = appHttpClient.get(appUrl, token, endpoint); + if (response.getCode() < 200 || response.getCode() >= 300) { + throw new IllegalStateException( + format("Error while executing a call to GitHub. Return code %s. Error message: %s.", response.getCode(), response.getContent().orElse(""))); + } + return response; + } +} diff --git a/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java new file mode 100644 index 00000000000..7ba266f4740 --- /dev/null +++ b/server/sonar-alm-client/src/main/java/org/sonar/alm/client/github/RatioBasedRateLimitChecker.java @@ -0,0 +1,57 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import com.google.common.annotations.VisibleForTesting; +import org.kohsuke.github.GHRateLimit; +import org.kohsuke.github.RateLimitChecker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.api.server.ServerSide; + +@ComputeEngineSide +@ServerSide +public class RatioBasedRateLimitChecker extends RateLimitChecker { + private static final Logger LOGGER = LoggerFactory.getLogger(RatioBasedRateLimitChecker.class); + + @VisibleForTesting + static final String RATE_RATIO_EXCEEDED_MESSAGE = "The GitHub API rate limit is almost reached. Pausing GitHub provisioning until the next rate limit reset. " + + "{} out of {} calls were used."; + + private static final int MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING = 90; + + @Override + public boolean checkRateLimit(GHRateLimit.Record rateLimitRecord, long count) throws InterruptedException { + int limit = rateLimitRecord.getLimit(); + int apiCallsUsed = limit - rateLimitRecord.getRemaining(); + double percentageOfCallsUsed = computePercentageOfCallsUsed(apiCallsUsed, limit); + LOGGER.debug("{} GitHub API calls used of {} available per hours", apiCallsUsed, limit); + if (percentageOfCallsUsed >= MAX_PERCENTAGE_OF_CALLS_FOR_PROVISIONING) { + LOGGER.warn(RATE_RATIO_EXCEEDED_MESSAGE, apiCallsUsed, limit); + return sleepUntilReset(rateLimitRecord); + } + return false; + } + + private static double computePercentageOfCallsUsed(int used, int limit) { + return (double) used * 100 / limit; + } +} diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java index 14ab8ce0654..c516cfafb35 100644 --- a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubApplicationClientImplTest.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import javax.annotation.Nullable; import org.junit.Before; import org.junit.ClassRule; @@ -52,6 +53,7 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.groups.Tuple.tuple; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -95,10 +97,12 @@ public class GithubApplicationClientImplTest { @ClassRule public static LogTester logTester = new LogTester().setLevel(LoggerLevel.WARN); - private GithubApplicationHttpClientImpl httpClient = mock(GithubApplicationHttpClientImpl.class); - private GithubAppSecurity appSecurity = mock(GithubAppSecurity.class); - private GithubAppConfiguration githubAppConfiguration = mock(GithubAppConfiguration.class); - private GitHubSettings gitHubSettings = mock(GitHubSettings.class); + private GithubApplicationHttpClientImpl httpClient = mock(); + private GithubAppSecurity appSecurity = mock(); + private GithubAppConfiguration githubAppConfiguration = mock(); + private GitHubSettings gitHubSettings = mock(); + + private GithubPaginatedHttpClient githubPaginatedHttpClient = mock(); private GithubApplicationClient underTest; private String appUrl = "Any URL"; @@ -106,7 +110,7 @@ public class GithubApplicationClientImplTest { @Before public void setup() { when(githubAppConfiguration.getApiEndpoint()).thenReturn(appUrl); - underTest = new GithubApplicationClientImpl(httpClient, appSecurity, gitHubSettings); + underTest = new GithubApplicationClientImpl(httpClient, appSecurity, gitHubSettings, githubPaginatedHttpClient); logTester.clear(); } @@ -512,11 +516,15 @@ public class GithubApplicationClientImplTest { assertThat(allOrgInstallations).isEmpty(); } + @SuppressWarnings("unchecked") private List getGithubAppInstallationsFromGithubResponse(String content) throws IOException { AppToken appToken = new AppToken(APP_JWT_TOKEN); when(appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey())).thenReturn(appToken); - when(httpClient.get(appUrl, appToken, "/app/installations")) - .thenReturn(new OkGetResponse(content)); + when(githubPaginatedHttpClient.get(eq(appUrl), eq(appToken), eq("/app/installations"), any())) + .thenAnswer(invocation -> { + Function> deserializingFunction = invocation.getArgument(3, Function.class); + return deserializingFunction.apply(content); + }); return underTest.getWhitelistedGithubAppInstallations(githubAppConfiguration); } @@ -524,8 +532,7 @@ public class GithubApplicationClientImplTest { public void getWhitelistedGithubAppInstallations_whenGithubReturnsError_shouldThrow() throws IOException { AppToken appToken = new AppToken(APP_JWT_TOKEN); when(appSecurity.createAppToken(githubAppConfiguration.getId(), githubAppConfiguration.getPrivateKey())).thenReturn(appToken); - when(httpClient.get(appUrl, appToken, "/app/installations")) - .thenReturn(new ErrorGetResponse()); + when(githubPaginatedHttpClient.get(any(),any(),any(),any())).thenThrow(new IOException("io exception")); assertThatThrownBy(() -> underTest.getWhitelistedGithubAppInstallations(githubAppConfiguration)) .isInstanceOf(IllegalStateException.class) diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java new file mode 100644 index 00000000000..8061e0c6dfc --- /dev/null +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/GithubPaginatedHttpClientImplTest.java @@ -0,0 +1,160 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.List; +import java.util.Optional; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.kohsuke.github.GHRateLimit; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.event.Level; +import org.sonar.alm.client.github.security.AccessToken; +import org.sonar.api.testfixtures.log.LogTester; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.sonar.alm.client.github.GithubApplicationHttpClient.GetResponse; + +@RunWith(MockitoJUnitRunner.class) +public class GithubPaginatedHttpClientImplTest { + + private static final String APP_URL = "https://github.com/"; + + private static final String ENDPOINT = "/test-endpoint"; + + private static final Type STRING_LIST_TYPE = TypeToken.getParameterized(List.class, String.class).getType(); + + private Gson gson = new Gson(); + + @Rule + public LogTester logTester = new LogTester(); + + @Mock + private AccessToken accessToken; + + @Mock + RatioBasedRateLimitChecker rateLimitChecker; + + @Mock + GithubApplicationHttpClient appHttpClient; + + @InjectMocks + private GithubPaginatedHttpClientImpl underTest; + + + @Test + public void get_whenNoPagination_ReturnsCorrectResponse() throws IOException { + + GetResponse response = mockResponseWithoutPagination("[\"result1\", \"result2\"]"); + when(appHttpClient.get(APP_URL, accessToken, ENDPOINT + "?per_page=100")).thenReturn(response); + + List results = underTest.get(APP_URL, accessToken, ENDPOINT, result -> gson.fromJson(result, STRING_LIST_TYPE)); + + assertThat(results) + .containsExactly("result1", "result2"); + } + + private static GetResponse mockResponseWithoutPagination(String content) { + GetResponse response = mock(GetResponse.class); + when(response.getCode()).thenReturn(200); + when(response.getContent()).thenReturn(Optional.of(content)); + return response; + } + + @Test + public void get_whenPaginationAndRateLimiting_returnsResponseFromAllPages() throws IOException, InterruptedException { + GetResponse response1 = mockResponseWithPaginationAndRateLimit("[\"result1\", \"result2\"]", "/next-endpoint"); + GetResponse response2 = mockResponseWithoutPagination("[\"result3\"]"); + when(appHttpClient.get(APP_URL, accessToken, ENDPOINT + "?per_page=100")).thenReturn(response1); + when(appHttpClient.get(APP_URL, accessToken, "/next-endpoint")).thenReturn(response2); + + List results = underTest.get(APP_URL, accessToken, ENDPOINT, result -> gson.fromJson(result, STRING_LIST_TYPE)); + + assertThat(results) + .containsExactly("result1", "result2", "result3"); + + ArgumentCaptor rateLimitRecordCaptor = ArgumentCaptor.forClass(GHRateLimit.Record.class); + verify(rateLimitChecker).checkRateLimit(rateLimitRecordCaptor.capture(), eq(0L)); + GHRateLimit.Record rateLimitRecord = rateLimitRecordCaptor.getValue(); + assertThat(rateLimitRecord.getLimit()).isEqualTo(10); + assertThat(rateLimitRecord.getRemaining()).isEqualTo(1); + assertThat(rateLimitRecord.getResetEpochSeconds()).isZero(); + } + + private static GetResponse mockResponseWithPaginationAndRateLimit(String content, String nextEndpoint) { + GetResponse response = mockResponseWithoutPagination(content); + when(response.getCode()).thenReturn(200); + when(response.getNextEndPoint()).thenReturn(Optional.of(nextEndpoint)); + when(response.getRateLimit()).thenReturn(new GithubApplicationHttpClient.RateLimit(1, 10, 0L)); + return response; + } + + @Test + public void get_whenGitHubReturnsNonSuccessCode_shouldThrow() throws IOException { + GetResponse response1 = mockResponseWithPaginationAndRateLimit("[\"result1\", \"result2\"]", "/next-endpoint"); + GetResponse response2 = mockFailedResponse("failed"); + when(appHttpClient.get(APP_URL, accessToken, ENDPOINT + "?per_page=100")).thenReturn(response1); + when(appHttpClient.get(APP_URL, accessToken, "/next-endpoint")).thenReturn(response2); + + assertThatIllegalStateException() + .isThrownBy(() -> underTest.get(APP_URL, accessToken, ENDPOINT, result -> gson.fromJson(result, STRING_LIST_TYPE))) + .withMessage("Error while executing a call to GitHub. Return code 400. Error message: failed."); + } + + private static GetResponse mockFailedResponse(String content) { + GetResponse response = mock(GetResponse.class); + when(response.getCode()).thenReturn(400); + when(response.getContent()).thenReturn(Optional.of(content)); + return response; + } + + @Test + public void getRepositoryTeams_whenRateLimitCheckerThrowsInterruptedException_shouldSucceed() throws IOException, InterruptedException { + GetResponse response1 = mockResponseWithPaginationAndRateLimit("[\"result1\", \"result2\"]", "/next-endpoint"); + GetResponse response2 = mockResponseWithoutPagination("[\"result3\"]"); + when(appHttpClient.get(APP_URL, accessToken, ENDPOINT + "?per_page=100")).thenReturn(response1); + when(appHttpClient.get(APP_URL, accessToken, "/next-endpoint")).thenReturn(response2); + doThrow(new InterruptedException("interrupted")).when(rateLimitChecker).checkRateLimit(any(GHRateLimit.Record.class), anyLong()); + + assertThatNoException() + .isThrownBy(() -> underTest.get(APP_URL, accessToken, ENDPOINT, result -> gson.fromJson(result, STRING_LIST_TYPE))); + + assertThat(logTester.logs()).hasSize(1); + assertThat(logTester.logs(Level.WARN)) + .containsExactly("Thread interrupted: interrupted"); + } +} diff --git a/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java new file mode 100644 index 00000000000..407e9056144 --- /dev/null +++ b/server/sonar-alm-client/src/test/java/org/sonar/alm/client/github/RatioBasedRateLimitCheckerTest.java @@ -0,0 +1,86 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.alm.client.github; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.sql.Date; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.kohsuke.github.GHRateLimit; +import org.mockito.Mockito; +import org.slf4j.event.Level; +import org.sonar.api.testfixtures.log.LogTester; + +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; +import static org.sonar.alm.client.github.RatioBasedRateLimitChecker.RATE_RATIO_EXCEEDED_MESSAGE; + +@RunWith(DataProviderRunner.class) +public class RatioBasedRateLimitCheckerTest { + + @Rule + public LogTester logTester = new LogTester(); + private static final long MILLIS_BEFORE_RESET = 100L; + RatioBasedRateLimitChecker ratioBasedRateLimitChecker = new RatioBasedRateLimitChecker(); + + @DataProvider + public static Object[][] rates() { + return new Object[][] { + {10000, 100000, false}, + {10000, 10000, false}, + {10000, 9999, false}, + {10000, 9900, false}, + {10000, 1001, false}, + {10000, 1000, true}, + {10000, 500, true}, + {10000, 0, true}, + }; + } + + @Test + @UseDataProvider("rates") + public void checkRateLimit(int limit, int remaining, boolean rateLimitShouldBeExceeded) throws InterruptedException { + GHRateLimit.Record record = Mockito.mock(GHRateLimit.Record.class); + when(record.getLimit()).thenReturn(limit); + when(record.getRemaining()).thenReturn(remaining); + when(record.getResetDate()).thenReturn(Date.from(Instant.now().plus(100, ChronoUnit.MILLIS))); + + long start = System.currentTimeMillis(); + boolean result = ratioBasedRateLimitChecker.checkRateLimit(record, 10); + long stop = System.currentTimeMillis(); + long totalTime = stop - start; + if (rateLimitShouldBeExceeded) { + assertThat(result).isTrue(); + assertThat(totalTime).isGreaterThanOrEqualTo(MILLIS_BEFORE_RESET - 10); + assertThat(logTester.logs(Level.WARN)).contains( + format(RATE_RATIO_EXCEEDED_MESSAGE.replaceAll("\\{\\}", "%s"), limit - remaining, limit)); + } else { + assertThat(result).isFalse(); + assertThat(totalTime).isLessThan(MILLIS_BEFORE_RESET); + assertThat(logTester.logs(Level.WARN)).isEmpty(); + } + } +} diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 9bd052497f4..d35a4504d3f 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -30,6 +30,8 @@ import org.sonar.alm.client.bitbucketserver.BitbucketServerSettingsValidator; import org.sonar.alm.client.github.GithubApplicationClientImpl; import org.sonar.alm.client.github.GithubApplicationHttpClientImpl; import org.sonar.alm.client.github.GithubGlobalSettingsValidator; +import org.sonar.alm.client.github.GithubPaginatedHttpClientImpl; +import org.sonar.alm.client.github.RatioBasedRateLimitChecker; import org.sonar.alm.client.github.config.GithubProvisioningConfigValidator; import org.sonar.alm.client.github.security.GithubAppSecurityImpl; import org.sonar.alm.client.gitlab.GitlabGlobalSettingsValidator; @@ -537,8 +539,10 @@ public class PlatformLevel4 extends PlatformLevel { CredentialsEncoderHelper.class, ImportHelper.class, ProjectKeyGenerator.class, + RatioBasedRateLimitChecker.class, GithubAppSecurityImpl.class, GithubApplicationClientImpl.class, + GithubPaginatedHttpClientImpl.class, GithubApplicationHttpClientImpl.class, GithubProvisioningConfigValidator.class, GithubProvisioningWs.class,