diff options
author | Lukasz Jarocki <lukasz.jarocki@sonarsource.com> | 2025-01-30 14:31:36 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2025-01-30 20:03:07 +0000 |
commit | c93341579d9ca9459962bdc7aa2ff7a9bdd025c5 (patch) | |
tree | e593f4b9da0a678f1eee910727263169773cdeac /server | |
parent | 8db63c7e8c138ef4df6787f1f36fbed1701d698d (diff) | |
download | sonarqube-c93341579d9ca9459962bdc7aa2ff7a9bdd025c5.tar.gz sonarqube-c93341579d9ca9459962bdc7aa2ff7a9bdd025c5.zip |
SONAR-23111 api/system/logs now correctly return logs for data center edition
Diffstat (limited to 'server')
13 files changed, 873 insertions, 79 deletions
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java index 1d9aaccaf70..05c2a8b7045 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java @@ -106,6 +106,7 @@ import org.sonar.server.issue.notification.NewIssuesNotificationHandler; import org.sonar.server.issue.workflow.FunctionExecutor; import org.sonar.server.issue.workflow.IssueWorkflow; import org.sonar.server.l18n.ServerI18n; +import org.sonar.server.log.DistributedServerLogging; import org.sonar.server.log.ServerLogging; import org.sonar.server.measure.index.ProjectMeasuresIndexer; import org.sonar.server.metric.IssueCountMetrics; @@ -417,7 +418,6 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { new ReportAnalysisFailureNotificationModule(), // System - ServerLogging.class, CEQueueStatusImpl.class, // SonarSource editions @@ -462,10 +462,12 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { // system info DbSection.class, + DistributedServerLogging.class, ProcessInfoProvider.class); } else { container.add( new CeCleaningModule(), + ServerLogging.class, StandaloneCeDistributedInformation.class); } } diff --git a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java index 90885d621c2..1ee435be9df 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java +++ b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java @@ -90,4 +90,12 @@ public class DistributedAnswer<T> { throw new IllegalStateException("Distributed cluster action timed out in cluster nodes " + timedOutMemberNames); } } + + /** + * Returns any answer. No guarantees are made on the order. Use this method if you only expect exactly one answer. + * @return the first answer, if any + */ + public Optional<T> getSingleAnswer() { + return answers.values().stream().findFirst(); + } } diff --git a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberSelectors.java b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberSelectors.java index f37fe594985..4b76f640cf8 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberSelectors.java +++ b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberSelectors.java @@ -19,6 +19,7 @@ */ package org.sonar.process.cluster.hz; +import com.hazelcast.cluster.Member; import com.hazelcast.cluster.MemberSelector; import java.util.List; import org.sonar.process.ProcessId; @@ -39,4 +40,8 @@ public class HazelcastMemberSelectors { return processIdList.contains(memberProcessId); }; } + + public static MemberSelector selectorForMember(Member member) { + return m -> m.getUuid().equals(member.getUuid()); + } } diff --git a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastObjects.java b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastObjects.java index 735919f852b..8624321fc43 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastObjects.java +++ b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastObjects.java @@ -53,6 +53,16 @@ public final class HazelcastObjects { */ public static final String SQ_HEALTH_STATE = "sq_health_state"; + /** + * Used in the header of HTTP call between the nodes to authenticate requests + */ + public static final String AUTH_SECRET = "AUTH_SECRET"; + + /** + * The key of replicated map holding the secrets. Used instead of CP Subsystem. + */ + public static final String SECRETS = "SECRETS"; + private HazelcastObjects() { // Holder for clustered objects } diff --git a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/DistributedAnswerTest.java b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/DistributedAnswerTest.java index e50776f6a99..295d8e00696 100644 --- a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/DistributedAnswerTest.java +++ b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/DistributedAnswerTest.java @@ -32,15 +32,14 @@ import static org.sonar.process.cluster.hz.HazelcastMember.Attribute.NODE_NAME; public class DistributedAnswerTest { - - private final Member member = newMember(UUID.randomUUID()); + private final Member member = newMember(uuidFromInt(9)); private final DistributedAnswer<String> underTest = new DistributedAnswer<>(); @Test public void getMembers_return_all_members() { underTest.setAnswer(member, "foo"); - underTest.setTimedOut(newMember(UUID.randomUUID())); - underTest.setFailed(newMember(UUID.randomUUID()), new IOException("BOOM")); + underTest.setTimedOut(newMember(uuidFromInt(0))); + underTest.setFailed(newMember(uuidFromInt(1)), new IOException("BOOM")); assertThat(underTest.getMembers()).hasSize(3); } @@ -100,7 +99,7 @@ public class DistributedAnswerTest { @Test public void propagateExceptions_does_nothing_if_no_errors() { - underTest.setAnswer(newMember(UUID.randomUUID()), "bar"); + underTest.setAnswer(newMember(uuidFromInt(3)), "bar"); // no errors underTest.propagateExceptions(); @@ -108,8 +107,8 @@ public class DistributedAnswerTest { @Test public void propagateExceptions_throws_ISE_if_at_least_one_timeout() { - UUID uuid = UUID.randomUUID(); - UUID otherUuid = UUID.randomUUID(); + UUID uuid = uuidFromInt(4); + UUID otherUuid = uuidFromInt(5); underTest.setAnswer(newMember(uuid), "baz"); underTest.setTimedOut(newMember(otherUuid)); @@ -121,8 +120,8 @@ public class DistributedAnswerTest { @Test public void propagateExceptions_throws_ISE_if_at_least_one_failure() { - UUID foo = UUID.randomUUID(); - UUID bar = UUID.randomUUID(); + UUID foo = uuidFromInt(0); + UUID bar = uuidFromInt(1); underTest.setAnswer(newMember(bar), "baz"); underTest.setFailed(newMember(foo), new IOException("BOOM")); @@ -132,6 +131,22 @@ public class DistributedAnswerTest { .hasMessage("Distributed cluster action in cluster nodes " + foo + " (other nodes may have timed out)"); } + @Test + public void getSingleAnswer_returns_first_answer() { + underTest.setAnswer(newMember(uuidFromInt(0)), "foo"); + + assertThat(underTest.getSingleAnswer()).isNotEmpty(); + } + + private UUID uuidFromInt(int digit) { + return UUID.fromString("00000000-0000-0000-0000-00000000000" + digit); + } + + @Test + public void getSingleAnswer_whenNoAnswer_returnEmptyOptional() { + assertThat(underTest.getSingleAnswer()).isEmpty(); + } + private static Member newMember(UUID uuid) { Member member = mock(Member.class); when(member.getUuid()).thenReturn(uuid); diff --git a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberSelectorsTest.java b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberSelectorsTest.java index e2352ade81b..cb51c6d5299 100644 --- a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberSelectorsTest.java +++ b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberSelectorsTest.java @@ -21,6 +21,7 @@ package org.sonar.process.cluster.hz; import com.hazelcast.cluster.Member; import com.hazelcast.cluster.MemberSelector; +import java.util.UUID; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -62,4 +63,37 @@ public class HazelcastMemberSelectorsTest { when(member.getAttribute(PROCESS_KEY.getKey())).thenReturn(APP.getKey()); assertThat(underTest.select(member)).isTrue(); } + + @Test + public void selectorForMember_whenUuidMatches_returnTrue() { + Member member = mock(); + Member member2 = mock(); + UUID uuid1 = uuidFromInt(0); + when(member.getUuid()).thenReturn(uuid1); + when(member2.getUuid()).thenReturn(uuid1); + + MemberSelector underTest = HazelcastMemberSelectors.selectorForMember(member); + boolean found = underTest.select(member2); + + assertThat(found).isTrue(); + } + + private UUID uuidFromInt(int digit) { + return UUID.fromString("00000000-0000-0000-0000-00000000000" + digit); + } + + @Test + public void selectorForMember_whenUuidDoesntMatch_returnTrue() { + Member member = mock(); + Member member2 = mock(); + UUID uuid1 = uuidFromInt(0); + UUID uuid2 = uuidFromInt(1); + when(member.getUuid()).thenReturn(uuid1); + when(member2.getUuid()).thenReturn(uuid2); + + MemberSelector underTest = HazelcastMemberSelectors.selectorForMember(member); + boolean found = underTest.select(member2); + + assertThat(found).isFalse(); + } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/log/DistributedServerLogging.java b/server/sonar-server-common/src/main/java/org/sonar/server/log/DistributedServerLogging.java new file mode 100644 index 00000000000..22dd9886b42 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/log/DistributedServerLogging.java @@ -0,0 +1,236 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.log; + +import com.google.common.annotations.VisibleForTesting; +import com.hazelcast.cluster.Member; +import jakarta.inject.Inject; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okio.Buffer; +import okio.BufferedSource; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.FileUtils; +import org.jetbrains.annotations.NotNull; +import org.sonar.api.config.Configuration; +import org.sonar.db.Database; +import org.sonar.process.ProcessId; +import org.sonar.process.cluster.hz.DistributedAnswer; +import org.sonar.process.cluster.hz.DistributedCall; +import org.sonar.process.cluster.hz.HazelcastMember; +import org.sonar.process.cluster.hz.HazelcastMemberSelectors; +import org.sonarqube.ws.client.OkHttpClientBuilder; + +import static org.sonar.process.cluster.hz.HazelcastMember.Attribute.PROCESS_KEY; +import static org.sonar.process.cluster.hz.HazelcastObjects.AUTH_SECRET; +import static org.sonar.process.cluster.hz.HazelcastObjects.SECRETS; + +public class DistributedServerLogging extends ServerLogging { + + public static final String NODE_TO_NODE_SECRET = "node_to_node_secret"; + private final HazelcastMember hazelcastMember; + private final OkHttpClient client; + + @Inject + public DistributedServerLogging(Configuration config, ServerProcessLogging serverProcessLogging, Database database, + HazelcastMember hazelcastMember) { + super(config, serverProcessLogging, database); + this.hazelcastMember = hazelcastMember; + this.client = new OkHttpClientBuilder() + .setConnectTimeoutMs(30000) + .setReadTimeoutMs(30000) + .setFollowRedirects(false) + .build(); + } + + @VisibleForTesting + public DistributedServerLogging(Configuration config, ServerProcessLogging serverProcessLogging, Database database, + HazelcastMember hazelcastMember, OkHttpClient client) { + super(config, serverProcessLogging, database); + this.hazelcastMember = hazelcastMember; + this.client = client; + } + + private WebAddress retrieveWebAddressOfAMember(Member member) { + try { + DistributedAnswer<WebAddress> answerWithAddress = hazelcastMember.call(askForWebAPIAddress(), + HazelcastMemberSelectors.selectorForMember(member), 5000L); + Optional<WebAddress> singleAnswer = answerWithAddress.getSingleAnswer(); + if (singleAnswer.isEmpty()) { + throw new IllegalStateException("No web API address found for member with UUID " + member.getUuid()); + } + return singleAnswer.get(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + + @Override + public boolean isValidNodeToNodeCall(Map<String, String> headers) { + String requestSecret = headers.get(NODE_TO_NODE_SECRET); + if (requestSecret == null || requestSecret.isEmpty()) { + return false; + } + Object savedSecret = tryGetSecretFromMap(); + return Objects.equals(savedSecret, requestSecret); + } + + private Object tryGetSecretFromMap() { + final int maxRetries = 5; + for (int i = 0; i < maxRetries; i++) { + Map<Object, Object> secrets = hazelcastMember.getReplicatedMap(SECRETS); + if (!secrets.containsKey(AUTH_SECRET)) { + sleep(2000); + continue; + } + return secrets.get(AUTH_SECRET); + } + return null; + } + + private static void sleep(int miliseconds) { + try { + Thread.sleep(miliseconds); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + + private static String generateOneTimeToken() { + SecureRandom random = new SecureRandom(); + byte[] randomBytes = new byte[20]; + random.nextBytes(randomBytes); + return Hex.encodeHexString(randomBytes); + } + + @Override + public File getDistributedLogs(String filePrefix, String logName) { + File zipFile = createZipFile(); + if (filePrefix.equals(ProcessId.ELASTICSEARCH.getLogFilenamePrefix())) { + // Elasticsearch logs are not distributed, something to implement in the future + throw new IllegalArgumentException("Elasticsearch logs are not distributed"); + } + + String oneTimeToken = generateOneTimeToken(); + setHazelcastAuthSecret(oneTimeToken); + try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile))) { + Set<Member> members = findMembersWithLogs(); + WebAddress localWebAPIAddress = askForWebAPIAddress().call(); + + // add data from other nodes + for (Member member : members) { + processMember(filePrefix, logName, member, localWebAPIAddress, oneTimeToken, out); + } + + // and add data from the current node + out.putNextEntry(createZipEntry(filePrefix, hazelcastMember.getUuid().toString())); + FileUtils.copyFile(getLogsForSingleNode(filePrefix), out); + } catch (Exception e) { + throw new IllegalStateException("Failed to collect logs from the nodes", e); + } finally { + resetHazelcastAuthSecret(); + } + return zipFile; + } + + private static File createZipFile() { + try { + Path tempDir = Files.createTempDirectory("logs"); + return new File(tempDir.toFile(), "logs.zip"); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @NotNull + private Set<Member> findMembersWithLogs() { + return hazelcastMember.getCluster().getMembers().stream() + .filter(member -> member.getAttribute(PROCESS_KEY.getKey()).equals(ProcessId.WEB_SERVER.getKey())) + .collect(Collectors.toSet()); + } + + private void processMember(String filePrefix, String logName, Member member, WebAddress localWebAPIAddress, + String oneTimeToken, ZipOutputStream out) throws IOException { + WebAddress addressAndPort = retrieveWebAddressOfAMember(member); + String address = addressAndPort.address; + int port = addressAndPort.port; + if (address.equals(localWebAPIAddress.address) && port == localWebAPIAddress.port) { + return; + } + + String url = String.format("http://%s:%d/api/system/logs?name=%s", address, port, logName); + Request request = new Request.Builder().url(url) + .addHeader(NODE_TO_NODE_SECRET, oneTimeToken) + .build(); + + try (Response response = client.newCall(request).execute()) { + out.putNextEntry(createZipEntry(filePrefix, member.getUuid().toString())); + addLogFromResponseToZip(response, out); + } + } + + @VisibleForTesting + static void addLogFromResponseToZip(Response response, ZipOutputStream out) throws IOException { + BufferedSource source = response.body().source(); + try (Buffer buffer = new Buffer()) { + while (!source.exhausted()) { + // we read up to 1MB at a time + long count = source.read(buffer, 1_048_576); + byte[] buf = new byte[(int) count]; + buffer.read(buf, 0, (int) count); + out.write(buf); + } + } + } + + private static ZipEntry createZipEntry(String filePrefix, String uuid) { + return new ZipEntry(filePrefix + "-" + uuid + ".log"); + } + + private void setHazelcastAuthSecret(String someSecret) { + hazelcastMember.getReplicatedMap(SECRETS).putIfAbsent(AUTH_SECRET, someSecret); + } + + private void resetHazelcastAuthSecret() { + hazelcastMember.getReplicatedMap(SECRETS).remove(AUTH_SECRET); + } + + private static DistributedCall<WebAddress> askForWebAPIAddress() { + return () -> new WebAddress(ServerLogging.getWebAPIAddressFromHazelcastQuery(), ServerLogging.getWebAPIPortFromHazelcastQuery()); + } + + public record WebAddress(String address, int port) implements Serializable { + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/log/ServerLogging.java b/server/sonar-server-common/src/main/java/org/sonar/server/log/ServerLogging.java index ae07279b670..21efc145238 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/log/ServerLogging.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/log/ServerLogging.java @@ -22,16 +22,26 @@ package org.sonar.server.log; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import com.google.common.annotations.VisibleForTesting; -import java.io.File; import jakarta.inject.Inject; -import org.sonar.api.Startable; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Stream; import org.slf4j.LoggerFactory; +import org.sonar.api.Startable; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.config.Configuration; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.api.utils.log.Loggers; import org.sonar.db.Database; +import org.sonar.process.ProcessProperties; import org.sonar.process.logging.LogbackHelper; import static org.sonar.api.utils.log.LoggerLevel.TRACE; @@ -40,7 +50,6 @@ import static org.sonar.process.ProcessProperties.Property.PATH_LOGS; @ServerSide @ComputeEngineSide public class ServerLogging implements Startable { - /** Used for Hazelcast's distributed queries in cluster mode */ private static ServerLogging instance; private final LogbackHelper helper; @@ -75,6 +84,17 @@ public class ServerLogging implements Startable { instance.changeLevel(level); } + public static int getWebAPIPortFromHazelcastQuery() { + Optional<String> port = instance.config.get(ProcessProperties.Property.WEB_PORT.getKey()); + return port.map(Integer::parseInt).orElse(9000); + } + + public static String getWebAPIAddressFromHazelcastQuery() { + return instance.config.get(ProcessProperties.Property.WEB_HOST.getKey()) + .orElseGet(() -> instance.config.get(ProcessProperties.Property.CLUSTER_NODE_HOST.getKey()) + .orElseThrow(() -> new IllegalStateException("No web host found in configuration"))); + } + public void changeLevel(LoggerLevel level) { Level logbackLevel = Level.toLevel(level.name()); database.enableSqlLogging(level == TRACE); @@ -93,4 +113,45 @@ public class ServerLogging implements Startable { return new File(config.get(PATH_LOGS.getKey()).get()); } + public Optional<Path> getLogFilePath(String filePrefix, File logsDir) throws IOException { + try (Stream<Path> stream = Files.list(Paths.get(logsDir.getPath()))) { + return stream + .filter(hasMatchingLogFiles(filePrefix)) + .max(Comparator.comparing(Path::toString)); + } + } + + public Predicate<Path> hasMatchingLogFiles(String filePrefix) { + return p -> { + String stringPath = p.getFileName().toString(); + return stringPath.startsWith(filePrefix) && stringPath.endsWith(".log"); + }; + } + + public File getLogsForSingleNode(String filePrefix) throws IOException { + File logsDir = getLogsDir(); + Optional<Path> path = getLogFilePath(filePrefix, logsDir); + + if (path.isEmpty()) { + return null; + } + + File file = new File(logsDir, path.get().getFileName().toString()); + + // filenames are defined in the enum LogProcess. Still to prevent any vulnerability, + // path is double-checked to prevent returning any file present on the file system. + if (file.exists() && file.getParentFile().equals(logsDir)) { + return file; + } else { + return null; + } + } + + public File getDistributedLogs(String filePrefix, String logName) { + throw new UnsupportedOperationException("This method should not be called on a standalone instance of SonarQube"); + } + + public boolean isValidNodeToNodeCall(Map<String, String> headers) { + return false; + } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/log/DistributedServerLoggingTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/log/DistributedServerLoggingTest.java new file mode 100644 index 00000000000..1a1745e5195 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/log/DistributedServerLoggingTest.java @@ -0,0 +1,187 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.log; + +import com.hazelcast.cluster.Cluster; +import com.hazelcast.cluster.Member; +import com.hazelcast.cp.IAtomicReference; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.zip.ZipOutputStream; +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Buffer; +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.db.Database; +import org.sonar.process.ProcessId; +import org.sonar.process.ProcessProperties; +import org.sonar.process.cluster.hz.DistributedAnswer; +import org.sonar.process.cluster.hz.HazelcastMember; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.sonar.process.ProcessProperties.Property.PATH_LOGS; +import static org.sonar.process.cluster.hz.HazelcastMember.Attribute.PROCESS_KEY; + +public class DistributedServerLoggingTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private final MapSettings settings = new MapSettings(); + private final ServerProcessLogging serverProcessLogging = mock(); + private final Database database = mock(); + private final HazelcastMember hazelcastMember = mock(); + private final OkHttpClient client = mock(); + + private final DistributedServerLogging underTest = new DistributedServerLogging(settings.asConfig(), serverProcessLogging, + database, hazelcastMember, client); + + private File dirWithLogs; + + @Before + public void before() throws InterruptedException, IOException { + Cluster cluster = mock(); + Member member1 = mock(), member2 = mock(); + Set<Member> members = Set.of(member1, member2); + when(hazelcastMember.getCluster()).thenReturn(cluster); + when(cluster.getMembers()).thenReturn(members); + when(member1.getAttribute(PROCESS_KEY.getKey())).thenReturn(ProcessId.WEB_SERVER.getKey()); + when(member2.getAttribute(PROCESS_KEY.getKey())).thenReturn(ProcessId.WEB_SERVER.getKey()); + + DistributedAnswer<Object> answer = mock(); + when(hazelcastMember.call(any(), any(), anyLong())).thenReturn(answer); + when(answer.getSingleAnswer()).thenReturn(Optional.of(new DistributedServerLogging.WebAddress("anyhost", 9000))); + when(hazelcastMember.getUuid()).thenReturn(UUID.fromString("00000000-0000-0000-0000-000000000000")); + when(member1.getUuid()).thenReturn(UUID.fromString("00000000-0000-0000-0000-000000000001")); + when(member2.getUuid()).thenReturn(UUID.fromString("00000000-0000-0000-0000-000000000002")); + + dirWithLogs = temp.newFolder(); + settings.setProperty(PATH_LOGS.getKey(), dirWithLogs.getAbsolutePath()); + settings.setProperty(ProcessProperties.Property.WEB_HOST.getKey(), "anyhost"); + + IAtomicReference<Object> reference = mock(); + when(hazelcastMember.getAtomicReference("AUTH_SECRET")).thenReturn(reference); + + underTest.start(); + } + + @Test + public void productionConstructor_doesNotThrowException() { + DistributedServerLogging distributedServerLogging = new DistributedServerLogging(settings.asConfig(), + serverProcessLogging, database, hazelcastMember); + + assertThat(distributedServerLogging).isNotNull(); + } + + @Test + public void isValidNodeToNodeCall_whenNodeToNodeSecretIsInvalid() { + IAtomicReference<Object> reference = mock(); + when(hazelcastMember.getAtomicReference("AUTH_SECRET")).thenReturn(reference); + when(reference.get()).thenReturn(""); + + boolean result = underTest.isValidNodeToNodeCall(Map.of("node_to_node_secret", "secret")); + + assertThat(result).isFalse(); + } + + @Test + public void isValidNodeToNodeCall_whenNodeToNodeSecretIsValid() { + when(hazelcastMember.getReplicatedMap("SECRETS")).thenReturn(Map.of("AUTH_SECRET", "secret")); + + boolean result = underTest.isValidNodeToNodeCall(Map.of("node_to_node_secret", "secret")); + + assertThat(result).isTrue(); + } + + @Test + public void addLogFromResponseToZip_whenSourceIsNotEmpty_writeToOutputStream() throws IOException { + ZipOutputStream zipOutputStream = mock(); + Response response = mock(); + ResponseBody responseBody = mock(); + when(response.body()).thenReturn(responseBody); + when(responseBody.source()).thenReturn(new Buffer().writeUtf8("response_body")); + + DistributedServerLogging.addLogFromResponseToZip(response, zipOutputStream); + + verify(zipOutputStream, atLeastOnce()).write(any(byte[].class)); + } + + @Test + public void getDistributedLogs_whenRetrieveWebAddressOfAMemberFails_throwException() throws InterruptedException { + when(hazelcastMember.call(any(), any(), anyLong())).thenThrow(new InterruptedException()); + + assertThatThrownBy(() -> underTest.getDistributedLogs("ce", "ce")) + .isInstanceOf(RuntimeException.class); + } + + @Test + public void getDistributedLogs_whenHttpCallFails_throwException() throws InterruptedException { + when(hazelcastMember.call(any(), any(), anyLong())).thenThrow(new InterruptedException()); + + assertThatThrownBy(() -> underTest.getDistributedLogs("ce", "ce")) + .isInstanceOf(IllegalStateException.class); + } + + @Test + public void getDistributedLogs_whenAnswerFromNodeIsEmpty_throwException() throws InterruptedException { + DistributedAnswer<Object> answer = mock(); + when(hazelcastMember.call(any(), any(), anyLong())).thenReturn(answer); + when(answer.getSingleAnswer()).thenReturn(Optional.empty()); + + assertThatThrownBy(() -> underTest.getDistributedLogs("ce", "ce")) + .isInstanceOf(IllegalStateException.class); + } + + @Test + public void getDistributedLogs_whenHttpCallPasses_returnFile() throws IOException { + Call call = mock(); + Response response = mock(); + ResponseBody responseBody = mock(); + when(client.newCall(any())).thenReturn(call); + when(call.execute()).thenReturn(response); + when(response.body()).thenReturn(responseBody); + when(responseBody.source()).thenReturn(new Buffer()); + + FileUtils.write(new File(dirWithLogs, "ce.log"), "ce_logs_data", Charset.defaultCharset()); + + File distributedLogs = underTest.getDistributedLogs("ce", "ce"); + + assertThat(distributedLogs).isNotNull(); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/log/ServerLoggingTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/log/ServerLoggingTest.java index 94e4c33cd44..65cfdcb6c4c 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/log/ServerLoggingTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/log/ServerLoggingTest.java @@ -25,6 +25,9 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Rule; import org.junit.Test; @@ -34,6 +37,7 @@ import org.sonar.api.config.internal.MapSettings; import org.sonar.api.testfixtures.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.db.Database; +import org.sonar.process.ProcessProperties; import org.sonar.process.logging.LogLevelConfig; import org.sonar.process.logging.LogbackHelper; @@ -58,11 +62,11 @@ public class ServerLoggingTest { public TemporaryFolder temp = new TemporaryFolder(); private final String rootLoggerName = RandomStringUtils.secure().nextAlphabetic(20); - private LogbackHelper logbackHelper = spy(new LogbackHelper()); - private MapSettings settings = new MapSettings(); + private final LogbackHelper logbackHelper = spy(new LogbackHelper()); + private final MapSettings settings = new MapSettings(); private final ServerProcessLogging serverProcessLogging = mock(ServerProcessLogging.class); private final Database database = mock(Database.class); - private ServerLogging underTest = new ServerLogging(logbackHelper, settings.asConfig(), serverProcessLogging, database); + private final ServerLogging underTest = new ServerLogging(logbackHelper, settings.asConfig(), serverProcessLogging, database); @Rule public LogTester logTester = new LogTester(); @@ -132,4 +136,99 @@ public class ServerLoggingTest { .isInstanceOf(IllegalArgumentException.class) .hasMessage("WARN log level is not supported (allowed levels are [TRACE, DEBUG, INFO])"); } + + @Test + public void getLogsForSingleNode_shouldReturnFile() throws IOException { + File dir = temp.newFolder(); + settings.setProperty(PATH_LOGS.getKey(), dir.getAbsolutePath()); + File file = new File(dir, "ce.log"); + file.createNewFile(); + + File ce = underTest.getLogsForSingleNode("ce"); + + assertThat(ce).isFile(); + } + + @Test + public void getLogFilePath_whenMatchingFileDoesNotExist_shouldReturnEmpty() throws IOException { + File dir = temp.newFolder(); + settings.setProperty(PATH_LOGS.getKey(), dir.getAbsolutePath()); + File file = new File(dir, "ce.log"); + file.createNewFile(); + + Optional<Path> path = underTest.getLogFilePath("web", dir); + + assertThat(path).isEmpty(); + } + + @Test + public void getLogFilePath_whenMatchingFileExists_shouldReturnPath() throws IOException { + File dir = temp.newFolder(); + settings.setProperty(PATH_LOGS.getKey(), dir.getAbsolutePath()); + File file = new File(dir, "web.log"); + file.createNewFile(); + + Optional<Path> path = underTest.getLogFilePath("web", dir); + + assertThat(path).isNotEmpty(); + } + + @Test + public void hasMatchingLogFiles_shouldReturnFalse() { + boolean result = underTest.hasMatchingLogFiles("ce").test(Path.of("web.log")); + + assertThat(result).isFalse(); + } + + @Test + public void hasMatchingLogFiles_shouldReturnTrue() { + boolean result = underTest.hasMatchingLogFiles("ce").test(Path.of("ce.log")); + + assertThat(result).isTrue(); + } + + @Test + public void getLogsForSingleNode_whenNoFiles_shouldReturnNull() throws IOException { + File dir = temp.newFolder(); + settings.setProperty(PATH_LOGS.getKey(), dir.getAbsolutePath()); + + File ce = underTest.getLogsForSingleNode("web"); + + assertThat(ce).isNull(); + } + + @Test + public void getDistributedLogs_shouldReturnException() { + assertThatThrownBy(() -> underTest.getDistributedLogs("a", "b")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("This method should not be called on a standalone instance of SonarQube"); + } + + @Test + public void isValidNodeToNodeCall_shouldReturnFalse() { + assertThat(underTest.isValidNodeToNodeCall(Map.of("node_to_node_secret", "secret"))).isFalse(); + } + + @Test + public void getWebAPIPortFromHazelcastQuery_shouldReturnPortByDefault() { + underTest.start(); + + assertThat(ServerLogging.getWebAPIPortFromHazelcastQuery()).isEqualTo(9000); + } + + @Test + public void getWebAPIPortFromHazelcastQuery_whenPortSpecified_shouldReturnPort() { + underTest.start(); + + settings.setProperty(ProcessProperties.Property.WEB_PORT.getKey(), "8000"); + assertThat(ServerLogging.getWebAPIPortFromHazelcastQuery()).isEqualTo(8000); + } + + @Test + public void getWebAPIAddressFromHazelcastQuery_whenSpecified_shouldReturnAddress() { + underTest.start(); + settings.setProperty(ProcessProperties.Property.CLUSTER_NODE_HOST.getKey(), "anyhost"); + + assertThat(ServerLogging.getWebAPIAddressFromHazelcastQuery()).isEqualTo("anyhost"); + } } diff --git a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/platform/ws/LogsActionIT.java b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/platform/ws/LogsActionIT.java index 5815acd5408..12055ed972b 100644 --- a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/platform/ws/LogsActionIT.java +++ b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/platform/ws/LogsActionIT.java @@ -19,15 +19,44 @@ */ package org.sonar.server.platform.ws; +import com.hazelcast.cluster.Cluster; +import com.hazelcast.cluster.Member; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Buffer; import org.apache.commons.io.FileUtils; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.Configuration; +import org.sonar.core.platform.EditionProvider; +import org.sonar.core.platform.PlatformEditionProvider; +import org.sonar.db.Database; +import org.sonar.process.ProcessId; +import org.sonar.process.ProcessProperties; +import org.sonar.process.cluster.hz.DistributedAnswer; +import org.sonar.process.cluster.hz.HazelcastMember; import org.sonar.server.exceptions.ForbiddenException; +import org.sonar.server.log.DistributedServerLogging; import org.sonar.server.log.ServerLogging; import org.sonar.server.tester.UserSessionRule; import org.sonar.server.ws.TestRequest; @@ -38,8 +67,11 @@ import org.sonarqube.ws.MediaTypes; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.sonar.process.ProcessProperties.Property.PATH_LOGS; public class LogsActionIT { @@ -48,10 +80,55 @@ public class LogsActionIT { @Rule public TemporaryFolder temp = new TemporaryFolder(); - private final ServerLogging serverLogging = mock(ServerLogging.class); - private final LogsAction underTest = new LogsAction(userSession, serverLogging); + private static final String DEFAULT_SECRET = "secret"; + + private final Configuration configuration = mock(); + private final Database database = mock(); + private final ServerLogging serverLogging = new ServerLogging(configuration, mock(), database); + private final PlatformEditionProvider editionProvider = mock(); + private final LogsAction underTest = new LogsAction(userSession, serverLogging, editionProvider); private final WsActionTester actionTester = new WsActionTester(underTest); + private final OkHttpClient client = mock(); + private final HazelcastMember hazelcastMember = mock(); + private final DistributedServerLogging distributedServerLogging = new DistributedServerLogging(configuration, mock(), database, hazelcastMember, client); + private final PlatformEditionProvider editionProviderDataCenter = mock(); + private final LogsAction underTestDataCenter = new LogsAction(userSession, distributedServerLogging, editionProviderDataCenter); + private final WsActionTester actionTesterDataCenter = new WsActionTester(underTestDataCenter); + + @Before + public void before() throws InterruptedException, IOException { + when(editionProvider.get()).thenReturn(Optional.of(EditionProvider.Edition.COMMUNITY)); + when(editionProviderDataCenter.get()).thenReturn(Optional.of(EditionProvider.Edition.DATACENTER)); + + Cluster cluster = mock(); + Member member1 = mock(), member2 = mock(); + Set<Member> members = Set.of(member1, member2); + Map<Object, Object> mapWithSecret = Map.of("AUTH_SECRET", DEFAULT_SECRET); + Call call = mock(); + DistributedAnswer answer = mock(); + Response response = mock(); + ResponseBody responseBody = mock(); + Buffer buffer = new Buffer(); + buffer.readFrom(new ByteArrayInputStream("exampleLog".getBytes(StandardCharsets.UTF_8))); + when(member1.getAttribute(any())).thenReturn(ProcessId.WEB_SERVER.getKey()); + when(member2.getAttribute(any())).thenReturn(ProcessId.WEB_SERVER.getKey()); + when(hazelcastMember.getReplicatedMap("SECRET")).thenReturn(mapWithSecret); + when(hazelcastMember.getCluster()).thenReturn(cluster); + when(cluster.getMembers()).thenReturn(members); + when(configuration.get(ProcessProperties.Property.WEB_HOST.getKey())).thenReturn(Optional.of("anyhost")); + when(hazelcastMember.call(any(), any(), anyLong())).thenReturn(answer); + when(answer.getSingleAnswer()).thenReturn(Optional.of(new DistributedServerLogging.WebAddress("anyhost", 9000))); + when(hazelcastMember.getUuid()).thenReturn(UUID.fromString("00000000-0000-0000-0000-000000000001")); + when(member1.getUuid()).thenReturn(UUID.fromString("00000000-0000-0000-0000-000000000002")); + when(member2.getUuid()).thenReturn(UUID.fromString("00000000-0000-0000-0000-000000000003")); + when(client.newCall(any())).thenReturn(call); + when(response.body()).thenReturn(responseBody); + when(responseBody.source()).thenReturn(buffer); + when(call.execute()).thenReturn(response); + distributedServerLogging.start(); + } + // values are lower-case and alphabetically ordered @Test public void possibleValues_shouldReturnPossibleLogFileValues() { @@ -87,19 +164,6 @@ public class LogsActionIT { } @Test - public void execute_whenUsingDeprecatedProcessParameter_shouldReturnCorrectLogs() throws IOException { - logInAsSystemAdministrator(); - - createAllLogsFiles(); - - TestResponse response = actionTester.newRequest() - .setParam("process", "deprecation") - .execute(); - assertThat(response.getMediaType()).isEqualTo(MediaTypes.TXT); - assertThat(response.getInput()).isEqualTo("{deprecation}"); - } - - @Test public void execute_whenFileDoesNotExist_shouldReturn404NotFound() throws IOException { logInAsSystemAdministrator(); @@ -125,6 +189,44 @@ public class LogsActionIT { } @Test + public void execute_whenDataCenterEdition_shouldReturnZipFile() throws IOException { + when(configuration.get(ProcessProperties.Property.WEB_PORT.getKey())).thenReturn(Optional.of("9000")); + when(configuration.get(ProcessProperties.Property.WEB_PORT.getKey())).thenReturn(Optional.of("9001")); + when(configuration.get(ProcessProperties.Property.WEB_PORT.getKey())).thenReturn(Optional.of("9002")); + logInAsSystemAdministrator(); + + createAllLogsFiles(); + + TestResponse response = actionTesterDataCenter.newRequest() + .setParam("name", "app") + .execute(); + assertThat(response.getMediaType()).isEqualTo(MediaTypes.ZIP); + assertZipContainsFiles(3, response, "sonar"); + } + + @Test + public void execute_whenDataCenterEdition_andOnlyOneNode_shouldReturnZipFileWithOneFile() throws IOException { + when(configuration.get(ProcessProperties.Property.WEB_PORT.getKey())).thenReturn(Optional.of("9000")); + logInAsSystemAdministrator(); + + createAllLogsFiles(); + + TestResponse response = actionTesterDataCenter.newRequest() + .setParam("name", "ce") + .execute(); + assertThat(response.getMediaType()).isEqualTo(MediaTypes.ZIP); + assertZipContainsFiles(1, response, "ce"); + } + + private void assertZipContainsFiles(int expectedFiles, TestResponse response, String expectedLog) throws IOException { + InputStream inputStream = response.getInputStream(); + List<File> files = extractFilesFromZip(inputStream); + + files.forEach(f -> assertThat(f.getName()).contains(expectedLog)); + assertThat(files).hasSize(expectedFiles); + } + + @Test public void execute_whenNumberRollingPolicy_shouldReturnLatestOnly() throws IOException { logInAsSystemAdministrator(); @@ -183,11 +285,60 @@ public class LogsActionIT { private File createLogsDir() throws IOException { File dir = temp.newFolder(); - when(serverLogging.getLogsDir()).thenReturn(dir); + when(configuration.get(PATH_LOGS.getKey())).thenReturn(Optional.of(dir.getAbsolutePath())); return dir; } private void logInAsSystemAdministrator() { userSession.logIn().setSystemAdministrator(); } + + + //helper methods for dealing with zip + public static List<File> extractFilesFromZip(InputStream zipInputStream) throws IOException { + List<File> extractedFiles = new ArrayList<>(); + File tempDir = Files.createTempDirectory("extractedZip").toFile(); + + try (ZipInputStream zis = new ZipInputStream(zipInputStream)) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + File newFile = newFile(tempDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + // fix for Windows-created archives + File parent = newFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + // write file content + try (FileOutputStream fos = new FileOutputStream(newFile)) { + byte[] buffer = new byte[1024]; + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } + } + } + extractedFiles.add(newFile); + } + } + return extractedFiles; + } + + private static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException { + File destFile = new File(destinationDir, zipEntry.getName()); + + String destDirPath = destinationDir.getCanonicalPath(); + String destFilePath = destFile.getCanonicalPath(); + + if (!destFilePath.startsWith(destDirPath + File.separator)) { + throw new IOException("Entry is outside of the target dir: " + zipEntry.getName()); + } + + return destFile; + } } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/LogsAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/LogsAction.java index 2d5b33a16e5..6e4ea46563b 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/LogsAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/LogsAction.java @@ -22,18 +22,13 @@ package org.sonar.server.platform.ws; import java.io.File; import java.io.IOException; import java.net.HttpURLConnection; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Comparator; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.Stream; import org.apache.commons.io.FileUtils; import org.sonar.api.server.ws.Change; import org.sonar.api.server.ws.Request; import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; +import org.sonar.core.platform.EditionProvider; +import org.sonar.core.platform.PlatformEditionProvider; import org.sonar.process.ProcessId; import org.sonar.server.log.ServerLogging; import org.sonar.server.user.UserSession; @@ -44,21 +39,18 @@ import static java.util.Arrays.stream; import static java.util.stream.Collectors.toList; public class LogsAction implements SystemWsAction { - /** - * @deprecated since 10.4, use {@link #NAME} instead. - */ - @Deprecated(since = "10.4", forRemoval = true) - private static final String PROCESS_PROPERTY = "process"; private static final String NAME = "name"; private static final String ACCESS_LOG = "access"; private static final String DEPRECATION_LOG = "deprecation"; private final UserSession userSession; private final ServerLogging serverLogging; + private final PlatformEditionProvider editionProvider; - public LogsAction(UserSession userSession, ServerLogging serverLogging) { + public LogsAction(UserSession userSession, ServerLogging serverLogging, PlatformEditionProvider editionProvider) { this.userSession = userSession; this.serverLogging = serverLogging; + this.editionProvider = editionProvider; } @Override @@ -73,13 +65,14 @@ public class LogsAction implements SystemWsAction { .setResponseExample(getClass().getResource("logs-example.log")) .setSince("5.2") .setChangelog( + new Change("2025.2", format("Added support for Data Center Edition for all possible values of '%s' except 'es'.", NAME)), + new Change("2025.2", "Removed deprecated 'process' property."), new Change("10.4", "Add support for deprecation logs in process property."), - new Change("10.4", format("Deprecate property '%s' in favor of '%s'.", PROCESS_PROPERTY, NAME))) + new Change("10.4", format("Deprecate property 'process' in favor of '%s'.", NAME))) .setHandler(this); action .createParam(NAME) - .setDeprecatedKey(PROCESS_PROPERTY, "10.4") .setPossibleValues(values) .setDefaultValue(ProcessId.APP.getKey()) .setSince("6.2") @@ -88,31 +81,36 @@ public class LogsAction implements SystemWsAction { @Override public void handle(Request wsRequest, Response wsResponse) throws Exception { - userSession.checkIsSystemAdministrator(); + boolean nodeToNodeCall = serverLogging.isValidNodeToNodeCall(wsRequest.getHeaders()); + if (!nodeToNodeCall) { + userSession.checkIsSystemAdministrator(); + } String logName = wsRequest.mandatoryParam(NAME); String filePrefix = getFilePrefix(logName); - File logsDir = serverLogging.getLogsDir(); - - Optional<Path> path = getLogFilePath(filePrefix, logsDir); - - if (path.isEmpty()) { - wsResponse.stream().setStatus(HttpURLConnection.HTTP_NOT_FOUND); - return; + if (!nodeToNodeCall && editionProvider.get().orElseThrow() == EditionProvider.Edition.DATACENTER) { + buildAndSendLogsForDataCenterEdition(wsResponse, filePrefix, logName); + } else { + buildAndSendLogsForSingleNode(wsResponse, filePrefix); } + } + + private void buildAndSendLogsForDataCenterEdition(Response wsResponse, String filePrefix, String logName) throws IOException { + File zipfile = serverLogging.getDistributedLogs(filePrefix, logName); - File file = new File(logsDir, path.get().getFileName().toString()); + wsResponse.stream().setMediaType(MediaTypes.ZIP); + FileUtils.copyFile(zipfile, wsResponse.stream().output()); + } - // filenames are defined in the enum LogProcess. Still to prevent any vulnerability, - // path is double-checked to prevent returning any file present on the file system. - if (file.exists() && file.getParentFile().equals(logsDir)) { + private void buildAndSendLogsForSingleNode(Response wsResponse, String filePrefix) throws IOException { + File file = serverLogging.getLogsForSingleNode(filePrefix); + if (file == null) { + wsResponse.stream().setStatus(HttpURLConnection.HTTP_NOT_FOUND); + } else { wsResponse.stream().setMediaType(MediaTypes.TXT); FileUtils.copyFile(file, wsResponse.stream().output()); - } else { - wsResponse.stream().setStatus(HttpURLConnection.HTTP_NOT_FOUND); } - } private static String getFilePrefix(String logName) { @@ -122,19 +120,4 @@ public class LogsAction implements SystemWsAction { default -> ProcessId.fromKey(logName).getLogFilenamePrefix(); }; } - - private static Optional<Path> getLogFilePath(String filePrefix, File logsDir) throws IOException { - try (Stream<Path> stream = Files.list(Paths.get(logsDir.getPath()))) { - return stream - .filter(hasMatchingLogFiles(filePrefix)) - .max(Comparator.comparing(Path::toString)); - } - } - - private static Predicate<Path> hasMatchingLogFiles(String filePrefix) { - return p -> { - String stringPath = p.getFileName().toString(); - return stringPath.startsWith(filePrefix) && stringPath.endsWith(".log"); - }; - } } diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index a84f68cb19e..a9c53d053e9 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -157,6 +157,7 @@ import org.sonar.server.issue.notification.NewModesNotificationsModule; import org.sonar.server.issue.ws.IssueWsModule; import org.sonar.server.language.LanguageValidation; import org.sonar.server.language.ws.LanguageWs; +import org.sonar.server.log.DistributedServerLogging; import org.sonar.server.log.ServerLogging; import org.sonar.server.loginmessage.LoginMessageFeature; import org.sonar.server.management.DelegatingManagedServices; @@ -330,7 +331,8 @@ public class PlatformLevel4 extends PlatformLevel { MetadataIndexImpl.class, EsDbCompatibilityImpl.class); - addIfCluster(new NodeHealthModule()); + addIfCluster(new NodeHealthModule(), + DistributedServerLogging.class); add( RuleDescriptionFormatter.class, @@ -571,7 +573,6 @@ public class PlatformLevel4 extends PlatformLevel { new ProjectAnalysisWsModule(), // System - ServerLogging.class, new ChangeLogLevelServiceModule(getWebServer()), new HealthCheckerModule(getWebServer()), new SystemWsModule(), @@ -744,6 +745,8 @@ public class PlatformLevel4 extends PlatformLevel { // system info add(new SystemInfoWriterModule(getWebServer())); + addIfStandalone(ServerLogging.class); + addAll(level4AddedComponents); } |