import org.sonar.server.component.ComponentService;
import org.sonar.server.computation.CeModule;
import org.sonar.server.computation.CeQueueModule;
-import org.sonar.server.computation.queue.PurgeCeActivities;
import org.sonar.server.computation.task.projectanalysis.ProjectAnalysisTaskModule;
import org.sonar.server.computation.taskprocessor.CeTaskProcessorModule;
import org.sonar.server.debt.DebtModelPluginRepository;
// RegisterIssueFilters.class, DB maintenance, responsibility of Web Server
// RenameIssueWidgets.class, UI related, anyway, DB maintenance, responsibility of Web Server
ServerLifecycleNotifier.class,
- PurgeCeActivities.class,
// DisplayLogOnDeprecatedProjects.class, responsibility of Web Server
// ClearRulesOverloadedDebt.class, DB maintenance, responsibility of Web Server
};
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
-import ch.qos.logback.core.Context;
-import ch.qos.logback.core.FileAppender;
-import ch.qos.logback.core.sift.AppenderFactory;
-import com.google.common.annotations.VisibleForTesting;
-import java.io.File;
-import org.sonar.ce.queue.CeTask;
-
-import static java.lang.String.format;
-
-/**
- * Creates a Logback appender for a Compute Engine task. See
- * http://logback.qos.ch/manual/loggingSeparation.html
- */
-public class CeFileAppenderFactory<E> implements AppenderFactory<E> {
-
- private static final String ENCODER_PATTERN = "%d{yyyy.MM.dd HH:mm:ss} %-5level [%logger{20}] %msg%n";
-
- private final File ceLogsDir;
-
- @VisibleForTesting
- CeFileAppenderFactory(File ceLogsDir) {
- this.ceLogsDir = ceLogsDir;
- }
-
- /**
- * @param context
- * @param discriminatingValue path of log file relative to the directory data/ce/logs
- * @see CeLogging#initForTask(CeTask)
- */
- @Override
- public FileAppender<E> buildAppender(Context context, String discriminatingValue) {
- PatternLayoutEncoder consoleEncoder = new PatternLayoutEncoder();
- consoleEncoder.setContext(context);
- consoleEncoder.setPattern(ENCODER_PATTERN);
- consoleEncoder.start();
- FileAppender appender = new FileAppender<>();
- appender.setContext(context);
- appender.setEncoder(consoleEncoder);
- appender.setName(format("ce-%s", discriminatingValue));
- appender.setFile(new File(ceLogsDir, discriminatingValue).getAbsolutePath());
- appender.start();
- return appender;
- }
-
-}
*/
package org.sonar.ce.log;
-import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
-import ch.qos.logback.classic.sift.MDCBasedDiscriminator;
-import ch.qos.logback.classic.sift.SiftingAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.FileAppender;
-import ch.qos.logback.core.sift.AppenderTracker;
-import ch.qos.logback.core.util.Duration;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import java.io.File;
-import java.util.Collections;
-import java.util.List;
import java.util.function.Supplier;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.comparator.LastModifiedFileComparator;
-import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.log4j.MDC;
-import org.sonar.api.config.Settings;
import org.sonar.ce.queue.CeTask;
import org.sonar.process.LogbackHelper;
-import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.FluentIterable.from;
-import static com.google.common.collect.Lists.newArrayList;
-import static java.lang.String.format;
-
/**
* Manages the logs written by Compute Engine:
* <ul>
*/
public class CeLogging {
- private static final long TIMEOUT_2_MINUTES = 1000 * 60 * 2L;
- private static final String CE_TASK_APPENDER_NAME = "ce_task";
private static final String CE_ACTIVITY_APPENDER_NAME = "ce_activity";
private static final String CE_ACTIVITY_FILE_NAME_PREFIX = "ce_activity";
private static final String CE_ACTIVITY_ENCODER_PATTERN = "%d{yyyy.MM.dd HH:mm:ss} %-5level [%logger{20}] %msg%n";
- // using 0L as timestamp when retrieving appender to stop it will make it instantly eligible for removal
- private static final long STOPPING_TRACKER_TIMESTAMP = 0L;
- @VisibleForTesting
- static final String MDC_LOG_PATH = "ceLogPath";
static final String MDC_CE_ACTIVITY_FLAG = "ceActivityFlag";
public static final String MAX_LOGS_PROPERTY = "sonar.ce.maxLogsPerTask";
- private final LogbackHelper helper = new LogbackHelper();
- private final File logsDir;
- private final Settings settings;
-
- public CeLogging(Settings settings) {
- String dataDir = settings.getString(ProcessProperties.PATH_DATA);
- checkArgument(dataDir != null, "Property %s is not set", ProcessProperties.PATH_DATA);
- this.logsDir = logsDirFromDataDir(new File(dataDir));
- this.settings = settings;
- }
-
- /**
- * Gets the log file of a given task. It may not exist if it
- * was purged or if the task does not exist.
- */
- public Optional<File> getFile(LogFileRef ref) {
- File logFile = new File(logsDir, ref.getRelativePath());
- if (logFile.exists()) {
- return Optional.of(logFile);
- }
- return Optional.absent();
- }
-
- public void deleteIfExists(LogFileRef ref) {
- File logFile = new File(logsDir, ref.getRelativePath());
- logFile.delete();
- }
-
- /**
- * Initialize logging of a Compute Engine task. Must be called
- * before first writing of log.
- * <p>After this method is executed, then Compute Engine logs are
- * written to a dedicated appender and are removed from sonar.log.</p>
- */
public void initForTask(CeTask task) {
- LogFileRef ref = LogFileRef.from(task);
- // Logback SiftingAppender requires to use a String, so
- // the path is put but not the object LogFileRef
- MDC.put(MDC_LOG_PATH, ref.getRelativePath());
+ // TODO put task UUID in MDC (cf. SONAR-7960)
}
- /**
- * Clean-up the logging of a task. Must be called after the last writing
- * of log.
- * <p>After this method is executed, then Compute Engine logs are
- * written to sonar.log only.</p>
- */
public void clearForTask() {
- String relativePath = (String) MDC.get(MDC_LOG_PATH);
- MDC.remove(MDC_LOG_PATH);
-
- if (relativePath != null) {
- stopAppender(relativePath);
- purgeDir(new File(logsDir, relativePath).getParentFile());
- }
+ // TODO clear task UUID from MDF (cf. SONAR-7960)
}
public static void logCeActivity(Runnable logProducer) {
return res;
}
- private void stopAppender(String relativePath) {
- Appender<ILoggingEvent> appender = helper.getRootContext().getLogger(Logger.ROOT_LOGGER_NAME).getAppender(CE_TASK_APPENDER_NAME);
- checkState(appender instanceof SiftingAppender, "Appender with name %s is null or not a SiftingAppender", CE_TASK_APPENDER_NAME);
- AppenderTracker<ILoggingEvent> ceAppender = ((SiftingAppender) appender).getAppenderTracker();
- ceAppender.getOrCreate(relativePath, STOPPING_TRACKER_TIMESTAMP).stop();
- }
-
- @VisibleForTesting
- void purgeDir(File dir) {
- if (dir.exists()) {
- int maxLogs = settings.getInt(MAX_LOGS_PROPERTY);
- if (maxLogs < 0) {
- throw new IllegalArgumentException(format("Property %s must be positive. Got: %d", MAX_LOGS_PROPERTY, maxLogs));
- }
- List<File> logFiles = newArrayList(FileUtils.listFiles(dir, FileFilterUtils.fileFileFilter(), FileFilterUtils.falseFileFilter()));
- if (logFiles.size() > maxLogs) {
- Collections.sort(logFiles, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR);
- for (File logFile : from(logFiles).limit(logFiles.size() - maxLogs)) {
- logFile.delete();
- }
- }
- }
- }
-
- /**
- * Directory which contains all the compute engine logs.
- * Log files must be persistent among server restarts and upgrades, so they are
- * stored into directory data/ but not into directories logs/ or temp/.
- * @return the non-null directory. It may not exist at startup.
- */
- static File logsDirFromDataDir(File dataDir) {
- return new File(dataDir, "ce/logs");
- }
-
- /**
- * Create Logback configuration for enabling sift appender.
- * A new log file is created for each task. It is based on MDC as long
- * as Compute Engine is not executed in its
- * own process but in the same process as web server.
- */
- public static Appender<ILoggingEvent> createPerTaskAppenderConfiguration(LoggerContext ctx, Props processProps) {
- File dataDir = new File(processProps.nonNullValue(ProcessProperties.PATH_DATA));
- File logsDir = logsDirFromDataDir(dataDir);
- return createPerTaskAppenderConfiguration(ctx, logsDir);
- }
-
- static SiftingAppender createPerTaskAppenderConfiguration(LoggerContext ctx, File logsDir) {
- SiftingAppender siftingAppender = new SiftingAppender();
- siftingAppender.addFilter(new CeTaskLogAcceptFilter<>());
- MDCBasedDiscriminator mdcDiscriminator = new MDCBasedDiscriminator();
- mdcDiscriminator.setContext(ctx);
- mdcDiscriminator.setKey(MDC_LOG_PATH);
- mdcDiscriminator.setDefaultValue("error");
- mdcDiscriminator.start();
- siftingAppender.setContext(ctx);
- siftingAppender.setDiscriminator(mdcDiscriminator);
- siftingAppender.setAppenderFactory(new CeFileAppenderFactory(logsDir));
- siftingAppender.setName(CE_TASK_APPENDER_NAME);
- siftingAppender.setTimeout(new Duration(TIMEOUT_2_MINUTES));
- siftingAppender.start();
- return siftingAppender;
- }
-
public static Appender<ILoggingEvent> createCeActivityAppenderConfiguration(LogbackHelper helper, LoggerContext ctx, Props processProps) {
LogbackHelper.RollingPolicy rollingPolicy = helper.createRollingPolicy(ctx, processProps, CE_ACTIVITY_FILE_NAME_PREFIX);
FileAppender<ILoggingEvent> fileAppender = rollingPolicy.createAppender(CE_ACTIVITY_APPENDER_NAME);
@Override
protected void configureAppenders(String logFormat, LoggerContext ctx, LogbackHelper helper, Props props) {
- ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(ctx, "CONSOLE", logFormat,
- new CeTaskLogDenyFilter<ILoggingEvent>(), new CeActivityLogDenyFilter());
+ ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(ctx, "CONSOLE", logFormat, new CeActivityLogDenyFilter());
ctx.getLogger(ROOT_LOGGER_NAME).addAppender(consoleAppender);
- ctx.getLogger(ROOT_LOGGER_NAME).addAppender(CeLogging.createPerTaskAppenderConfiguration(ctx, props));
ctx.getLogger(ROOT_LOGGER_NAME).addAppender(CeLogging.createCeActivityAppenderConfiguration(helper, ctx, props));
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import ch.qos.logback.core.filter.Filter;
-import ch.qos.logback.core.spi.FilterReply;
-import org.slf4j.MDC;
-
-/**
- * Keeps only the Compute Engine activity logs.
- */
-public class CeTaskLogAcceptFilter<E> extends Filter<E> {
-
- @Override
- public FilterReply decide(E o) {
- return MDC.get(CeLogging.MDC_LOG_PATH) == null ? FilterReply.DENY : FilterReply.ACCEPT;
- }
-
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import ch.qos.logback.core.filter.Filter;
-import ch.qos.logback.core.spi.FilterReply;
-import org.slf4j.MDC;
-
-/**
- * Filters out the Compute Engine logs.
- */
-public class CeTaskLogDenyFilter<E> extends Filter<E> {
-
- @Override
- public FilterReply decide(E o) {
- return MDC.get(CeLogging.MDC_LOG_PATH) == null ? FilterReply.ACCEPT : FilterReply.DENY;
- }
-
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.util.regex.Pattern;
-import javax.annotation.CheckForNull;
-import javax.annotation.Nullable;
-import org.sonar.db.ce.CeActivityDto;
-import org.sonar.db.ce.CeQueueDto;
-import org.sonar.ce.queue.CeTask;
-
-import static java.lang.String.format;
-
-public class LogFileRef {
-
- // restricted white-list for now
- private static final Pattern FILENAME_PATTERN = Pattern.compile("^[\\w\\-]*$");
- private final String taskType;
- private final String taskUuid;
-
- @CheckForNull
- private final String componentUuid;
-
- public LogFileRef(String taskType, String taskUuid, @Nullable String componentUuid) {
- this.taskType = requireValidFilename(taskType);
- this.taskUuid = requireValidFilename(taskUuid);
- this.componentUuid = requireValidFilename(componentUuid);
- }
-
- @VisibleForTesting
- @CheckForNull
- static String requireValidFilename(@Nullable String s) {
- if (s != null && !FILENAME_PATTERN.matcher(s).matches()) {
- throw new IllegalArgumentException(String.format("'%s' is not a valid filename for Compute Engine logs", s));
- }
- return s;
- }
-
- /**
- * Path relative to the CE logs directory
- */
- public String getRelativePath() {
- if (componentUuid == null) {
- return format("%s/%s.log", taskType, taskUuid);
- }
- return format("%s/%s/%s.log", taskType, componentUuid, taskUuid);
- }
-
- @Override
- public boolean equals(@Nullable Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- LogFileRef that = (LogFileRef) o;
- if (!taskType.equals(that.taskType)) {
- return false;
- }
- if (!taskUuid.equals(that.taskUuid)) {
- return false;
- }
- return componentUuid == null ? (that.componentUuid == null) : componentUuid.equals(that.componentUuid);
-
- }
-
- @Override
- public int hashCode() {
- int result = taskType.hashCode();
- result = 31 * result + taskUuid.hashCode();
- result = 31 * result + (componentUuid != null ? componentUuid.hashCode() : 0);
- return result;
- }
-
- public static LogFileRef from(CeActivityDto dto) {
- return new LogFileRef(dto.getTaskType(), dto.getUuid(), dto.getComponentUuid());
- }
-
- public static LogFileRef from(CeQueueDto dto) {
- return new LogFileRef(dto.getTaskType(), dto.getUuid(), dto.getComponentUuid());
- }
-
- public static LogFileRef from(CeTask task) {
- return new LogFileRef(task.getType(), task.getUuid(), task.getComponentUuid());
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.computation.queue;
-
-import java.util.Calendar;
-import java.util.List;
-import org.sonar.api.platform.Server;
-import org.sonar.api.platform.ServerStartHandler;
-import org.sonar.api.ce.ComputeEngineSide;
-import org.sonar.api.utils.System2;
-import org.sonar.api.utils.log.Logger;
-import org.sonar.api.utils.log.Loggers;
-import org.sonar.ce.log.CeLogging;
-import org.sonar.ce.log.LogFileRef;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ce.CeActivityDto;
-
-@ComputeEngineSide
-public class PurgeCeActivities implements ServerStartHandler {
-
- private static final Logger LOGGER = Loggers.get(PurgeCeActivities.class);
-
- private final DbClient dbClient;
- private final System2 system2;
- private final CeLogging ceLogging;
-
- public PurgeCeActivities(DbClient dbClient, System2 system2, CeLogging ceLogging) {
- this.dbClient = dbClient;
- this.system2 = system2;
- this.ceLogging = ceLogging;
- }
-
- @Override
- public void onServerStart(Server server) {
- DbSession dbSession = dbClient.openSession(true);
- try {
- Calendar sixMonthsAgo = Calendar.getInstance();
- sixMonthsAgo.setTimeInMillis(system2.now());
- sixMonthsAgo.add(Calendar.DATE, -180);
-
- LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime());
- List<CeActivityDto> dtos = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTimeInMillis());
- for (CeActivityDto dto : dtos) {
- dbClient.ceActivityDao().deleteByUuid(dbSession, dto.getUuid());
- ceLogging.deleteIfExists(LogFileRef.from(dto));
- }
- dbSession.commit();
-
- } finally {
- dbClient.closeSession(dbSession);
- }
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import ch.qos.logback.classic.LoggerContext;
-import ch.qos.logback.core.FileAppender;
-import java.io.File;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class CeFileAppenderFactoryTest {
-
- @Rule
- public TemporaryFolder temp = new TemporaryFolder();
-
- @Test
- public void buildAppender() throws Exception {
- File logsDir = temp.newFolder();
- CeFileAppenderFactory factory = new CeFileAppenderFactory(logsDir);
-
- FileAppender underTest = factory.buildAppender(new LoggerContext(), "uuid_1.log");
-
- assertThat(new File(underTest.getFile())).isEqualTo(new File(logsDir, "uuid_1.log"));
-
- }
-}
*/
package org.sonar.ce.log;
-import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
-import ch.qos.logback.classic.sift.SiftingAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.FileAppender;
import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.joran.spi.JoranException;
-import com.google.common.base.Optional;
import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
-import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
-import org.sonar.api.config.Settings;
-import org.sonar.ce.queue.CeTask;
import org.sonar.process.LogbackHelper;
import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;
-import static java.lang.String.format;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import static org.sonar.ce.log.CeLogging.MDC_CE_ACTIVITY_FLAG;
-import static org.sonar.ce.log.CeLogging.MDC_LOG_PATH;
public class CeLoggingTest {
public TemporaryFolder temp = new TemporaryFolder();
private LogbackHelper helper = new LogbackHelper();
- private File dataDir;
private File logDir;
@Before
public void setUp() throws Exception {
- this.dataDir = temp.newFolder();
this.logDir = temp.newFolder();
}
MDC.clear();
}
- @Test
- public void getFile() throws IOException {
- Settings settings = newSettings(dataDir, 10);
-
- CeLogging underTest = new CeLogging(settings);
- LogFileRef ref = new LogFileRef("TYPE1", "TASK1", "COMPONENT1");
-
- // file does not exist
- Optional<File> file = underTest.getFile(ref);
- assertThat(file.isPresent()).isFalse();
-
- File logFile = new File(dataDir, "ce/logs/" + ref.getRelativePath());
- FileUtils.touch(logFile);
- file = underTest.getFile(ref);
- assertThat(file.isPresent()).isTrue();
- assertThat(file.get()).isEqualTo(logFile);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void fail_if_data_dir_is_not_set() {
- new CeLogging(new Settings());
- }
-
- @Test
- public void initForTask_adds_path_of_ce_log_file_in_MDC() throws IOException {
- CeLogging underTest = new CeLogging(newSettings(dataDir, 5));
-
- CeTask task = createCeTask("TYPE1", "U1");
- underTest.initForTask(task);
- assertThat(MDC.get(MDC_LOG_PATH)).isNotEmpty().isEqualTo(LogFileRef.from(task).getRelativePath());
- }
-
- @Test
- public void clearForTask_throws_ISE_if_CE_appender_is_not_configured() throws IOException {
- CeLogging underTest = new CeLogging(newSettings(dataDir, 5));
-
- CeTask task = createCeTask("TYPE1", "U1");
- underTest.initForTask(task);
-
- expectedException.expect(IllegalStateException.class);
- expectedException.expectMessage("Appender with name ce_task is null or not a SiftingAppender");
-
- underTest.clearForTask();
- }
-
- @Test
- public void clearForTask_throws_ISE_if_CE_appender_is_not_a_SiftingAppender() throws IOException {
- Appender<ILoggingEvent> mockCeAppender = mock(Appender.class);
- when(mockCeAppender.getName()).thenReturn("ce");
- helper.getRootContext().getLogger(Logger.ROOT_LOGGER_NAME).addAppender(mockCeAppender);
-
- CeLogging underTest = new CeLogging(newSettings(dataDir, 5));
-
- CeTask task = createCeTask("TYPE1", "U1");
- underTest.initForTask(task);
-
- expectedException.expect(IllegalStateException.class);
- expectedException.expectMessage("Appender with name ce_task is null or not a SiftingAppender");
-
- underTest.clearForTask();
- }
-
- @Test
- public void clearForTask_clears_MDC() throws IOException {
- setupCeTaskAppender();
-
- CeLogging underTest = new CeLogging(newSettings(dataDir, 5));
-
- CeTask task = createCeTask("TYPE1", "U1");
- underTest.initForTask(task);
- assertThat(MDC.get(MDC_LOG_PATH)).isNotEmpty().isEqualTo(LogFileRef.from(task).getRelativePath());
-
- underTest.clearForTask();
- assertThat(MDC.get(MDC_LOG_PATH)).isNull();
- }
-
- @Test
- public void cleanForTask_stops_only_appender_for_MDC_value() throws IOException {
- Logger rootLogger = setupCeTaskAppender();
-
- CeLogging underTest = new CeLogging(newSettings(dataDir, 5));
-
- // init MDC
- underTest.initForTask(createCeTask("TYPE1", "U1"));
- verifyNoAppender(rootLogger);
-
- // logging will create and start the appender
- LoggerFactory.getLogger(getClass()).info("some log!");
- verifyAllAppenderStarted(rootLogger, 1);
-
- // init MDC and create appender for another task
- // (in the same thread, which should not happen, but it's good enough for our test)
- CeTask ceTask = createCeTask("TYPE1", "U2");
- underTest.initForTask(ceTask);
- LoggerFactory.getLogger(getClass()).info("some other log!");
- verifyAllAppenderStarted(rootLogger, 2);
-
- // stop appender which is currently referenced in MDC
- underTest.clearForTask();
-
- Appender appender = verifySingleAppenderIsStopped(rootLogger, 2);
- assertThat(appender.getName()).isEqualTo("ce-" + LogFileRef.from(ceTask).getRelativePath());
- }
-
- @Test
- public void delete_oldest_files_of_same_directory_to_keep_only_max_allowed_files() throws IOException {
- for (int i = 1; i <= 5; i++) {
- File file = new File(dataDir, format("U%d.log", i));
- FileUtils.touch(file);
- // see javadoc: "all platforms support file-modification times to the nearest second,
- // but some provide more precision" --> increment by second, not by millisecond
- file.setLastModified(1_450_000_000_000L + i * 1000);
- }
- assertThat(dataDir.listFiles()).hasSize(5);
-
- // keep 3 files in each dir
- CeLogging underTest = new CeLogging(newSettings(dataDir, 3));
- underTest.purgeDir(dataDir);
-
- assertThat(dataDir.listFiles()).hasSize(3);
- assertThat(dataDir.listFiles()).extracting("name")
- .containsOnly("U3.log", "U4.log", "U5.log");
- }
-
- @Test
- public void do_not_delete_files_if_dir_has_less_files_than_max_allowed() throws IOException {
- FileUtils.touch(new File(dataDir, "U1.log"));
-
- CeLogging underTest = new CeLogging(newSettings(dataDir, 5));
- underTest.purgeDir(dataDir);
-
- assertThat(dataDir.listFiles()).extracting("name").containsOnly("U1.log");
- }
-
- @Test
- public void do_not_keep_any_logs() throws IOException {
- FileUtils.touch(new File(dataDir, "U1.log"));
-
- CeLogging underTest = new CeLogging(newSettings(dataDir, 0));
- underTest.purgeDir(dataDir);
-
- assertThat(dataDir.listFiles()).isEmpty();
- }
-
- @Test
- public void fail_if_max_logs_settings_is_negative() throws IOException {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Property sonar.ce.maxLogsPerTask must be positive. Got: -1");
-
- Settings settings = newSettings(dataDir, -1);
- CeLogging logging = new CeLogging(settings);
- logging.purgeDir(dataDir);
- }
-
- @Test
- public void createTaskConfiguration() {
- SiftingAppender siftingAppender = CeLogging.createPerTaskAppenderConfiguration(new LoggerContext(), dataDir);
-
- // filter on CE logs
- List<Filter<ILoggingEvent>> filters = siftingAppender.getCopyOfAttachedFiltersList();
- assertThat(filters).hasSize(1);
- assertThat(filters.get(0)).isInstanceOf(CeTaskLogAcceptFilter.class);
-
- assertThat(siftingAppender.getDiscriminator().getKey()).isEqualTo(MDC_LOG_PATH);
- assertThat(siftingAppender.getTimeout().getMilliseconds()).isEqualTo(1000 * 60 * 2);
- }
-
@Test
public void createCeConfigurationConfiguration_fails_if_log_directory_is_not_set_in_Props() {
LogbackHelper helper = new LogbackHelper();
assertThat(called.get()).isTrue();
}
- private Logger setupCeTaskAppender() {
- Logger rootLogger = helper.getRootContext().getLogger(Logger.ROOT_LOGGER_NAME);
- rootLogger.addAppender(CeLogging.createPerTaskAppenderConfiguration(helper.getRootContext(), dataDir));
- return rootLogger;
- }
-
- private void verifyNoAppender(Logger rootLogger) {
- Collection<Appender<ILoggingEvent>> allAppenders = getAllAppenders(rootLogger);
- assertThat(allAppenders).isEmpty();
- }
-
- private void verifyAllAppenderStarted(Logger rootLogger, int expectedSize) {
- Collection<Appender<ILoggingEvent>> allAppenders = getAllAppenders(rootLogger);
- assertThat(allAppenders).hasSize(expectedSize);
- for (Appender<ILoggingEvent> appender : allAppenders) {
- assertThat(appender.isStarted()).isTrue();
- }
- }
-
- private Appender verifySingleAppenderIsStopped(Logger rootLogger, int expectedSize) {
- Collection<Appender<ILoggingEvent>> allAppenders = getAllAppenders(rootLogger);
- assertThat(allAppenders).hasSize(expectedSize);
- Appender res = null;
- for (Appender<ILoggingEvent> appender : allAppenders) {
- if (!appender.isStarted()) {
- assertThat(res).describedAs("More than one appender found stopped").isNull();
- res = appender;
- }
- }
- assertThat(res).describedAs("There should be one stopped appender").isNotNull();
- return res;
- }
-
- private Collection<Appender<ILoggingEvent>> getAllAppenders(Logger rootLogger) {
- Appender<ILoggingEvent> ceAppender = rootLogger.getAppender("ce_task");
- assertThat(ceAppender).isInstanceOf(SiftingAppender.class);
- return ((SiftingAppender) ceAppender).getAppenderTracker().allComponents();
- }
-
- private static Settings newSettings(File dataDir, int maxLogs) {
- Settings settings = new Settings();
- settings.setProperty(ProcessProperties.PATH_DATA, dataDir.getAbsolutePath());
- settings.setProperty(CeLogging.MAX_LOGS_PROPERTY, maxLogs);
- return settings;
- }
-
- private static CeTask createCeTask(String type, String uuid) {
- return new CeTask.Builder().setType(type).setUuid(uuid).build();
- }
}
private CeProcessLogging underTest = new CeProcessLogging();
/**
- * Path to data dir must be set for Compute Engine logging.
- * @see CeFileAppenderFactory
* Path to log dir must be set for Compute Engine Activity logging.
* @see org.sonar.ce.log.CeLogging#createCeActivityAppenderConfiguration(LogbackHelper, LoggerContext, Props)
*/
@Before
public void setUp() throws IOException {
- props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import ch.qos.logback.core.filter.Filter;
-import ch.qos.logback.core.spi.FilterReply;
-import org.apache.log4j.MDC;
-import org.junit.After;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class CeTaskLogAcceptFilterTest {
-
- private static final Object UNUSED = "";
-
- Filter underTest = new CeTaskLogAcceptFilter();
-
- @After
- public void tearDown() {
- MDC.clear();
- }
-
- @Test
- public void reject_non_ce_logs() {
- assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.DENY);
- }
-
- @Test
- public void accept_ce_logs() {
- MDC.put(CeLogging.MDC_LOG_PATH, "abc.log");
- assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.ACCEPT);
- }
-
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import ch.qos.logback.core.filter.Filter;
-import ch.qos.logback.core.spi.FilterReply;
-import org.apache.log4j.MDC;
-import org.junit.After;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class CeTaskLogDenyFilterTest {
-
- private static final Object UNUSED = "";
-
- Filter underTest = new CeTaskLogDenyFilter();
-
- @After
- public void tearDown() {
- MDC.clear();
- }
-
- @Test
- public void accept_non_ce_logs() {
- assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.ACCEPT);
- }
-
- @Test
- public void deny_ce_logs() {
- MDC.put(CeLogging.MDC_LOG_PATH, "abc.log");
- assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.DENY);
- }
-
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.ce.log;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.sonar.ce.log.LogFileRef;
-import org.sonar.db.ce.CeTaskTypes;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class LogFileRefTest {
-
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- @Test
- public void equals_hashCode() {
- LogFileRef ref1 = new LogFileRef(CeTaskTypes.REPORT, "UUID_1", "COMPONENT_1");
- LogFileRef ref1bis = new LogFileRef(CeTaskTypes.REPORT, "UUID_1", "COMPONENT_1");
- LogFileRef ref2 = new LogFileRef(CeTaskTypes.REPORT, "UUID_2", "COMPONENT_1");
-
- assertThat(ref1.equals(ref1)).isTrue();
- assertThat(ref1.equals(ref1bis)).isTrue();
- assertThat(ref1.equals(ref2)).isFalse();
- assertThat(ref1.equals(null)).isFalse();
- assertThat(ref1.equals("UUID_1")).isFalse();
-
- assertThat(ref1.hashCode()).isEqualTo(ref1bis.hashCode());
- }
-
- @Test
- public void getRelativePath() {
- assertThat(new LogFileRef("TYPE_1", "UUID_1", "COMPONENT_1").getRelativePath()).isEqualTo("TYPE_1/COMPONENT_1/UUID_1.log");
- assertThat(new LogFileRef("TYPE_1", "UUID_1", null).getRelativePath()).isEqualTo("TYPE_1/UUID_1.log");
- }
-
- @Test
- public void do_not_accept_invalid_task_type() {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("'foo/bar' is not a valid filename for Compute Engine logs");
-
- new LogFileRef("foo/bar", "UUID", null);
- }
-
- @Test
- public void do_not_accept_invalid_uuid() {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("'foo/bar' is not a valid filename for Compute Engine logs");
-
- new LogFileRef("REPORT", "foo/bar", null);
- }
-
- @Test
- public void do_not_accept_invalid_component_uuid() {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("'foo/bar' is not a valid filename for Compute Engine logs");
-
- new LogFileRef("REPORT", "UUID", "foo/bar");
- }
-
- @Test
- public void filename_must_support_uuid() {
- String uuid = "AU-Tpxb-_iU5OvuD2FLy";
- assertThat(LogFileRef.requireValidFilename(uuid)).isEqualTo(uuid);
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2016 SonarSource SA
- * mailto:contact AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.computation;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.sonar.api.platform.Server;
-import org.sonar.api.utils.internal.TestSystem2;
-import org.sonar.db.DbTester;
-import org.sonar.db.ce.CeActivityDto;
-import org.sonar.db.ce.CeQueueDto;
-import org.sonar.db.ce.CeTaskTypes;
-import org.sonar.ce.log.CeLogging;
-import org.sonar.ce.log.LogFileRef;
-import org.sonar.server.computation.queue.PurgeCeActivities;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-
-public class PurgeCeActivitiesTest {
-
- TestSystem2 system2 = new TestSystem2();
-
- @Rule
- public DbTester dbTester = DbTester.create(system2);
- CeLogging ceLogging = mock(CeLogging.class);
- PurgeCeActivities underTest = new PurgeCeActivities(dbTester.getDbClient(), system2, ceLogging);
-
- @Test
- public void delete_older_than_6_months() throws Exception {
- insertWithDate("VERY_OLD", 1_000_000_000_000L);
- insertWithDate("RECENT", 1_500_000_000_000L);
- system2.setNow(1_500_000_000_100L);
-
- underTest.onServerStart(mock(Server.class));
-
- assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "VERY_OLD").isPresent()).isFalse();
- assertThat(dbTester.getDbClient().ceActivityDao().selectByUuid(dbTester.getSession(), "RECENT").isPresent()).isTrue();
- verify(ceLogging).deleteIfExists(new LogFileRef(CeTaskTypes.REPORT, "VERY_OLD", null));
- verify(ceLogging, never()).deleteIfExists(new LogFileRef(CeTaskTypes.REPORT, "RECENT", null));
- }
-
- private void insertWithDate(String uuid, long date) {
- CeQueueDto queueDto = new CeQueueDto();
- queueDto.setUuid(uuid);
- queueDto.setTaskType(CeTaskTypes.REPORT);
-
- CeActivityDto dto = new CeActivityDto(queueDto);
- dto.setStatus(CeActivityDto.Status.SUCCESS);
- system2.setNow(date);
- dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto);
- dbTester.getSession().commit();
- }
-}
projects / sonarqube.git / commitdiff