Selaa lähdekoodia

SONAR-6799 SONAR-6232 compute engine logs

- add WS api/ce/logs
- add boolean field 'logs' to the responses of the WS that return tasks
- purge log files (max nb of files per project can be configured)
tags/5.2-RC1
Simon Brandhof 8 vuotta sitten
vanhempi
commit
7ed36fc7a3
41 muutettua tiedostoa jossa 1608 lisäystä ja 56 poistoa
  1. 7
    3
      server/sonar-process/src/main/java/org/sonar/process/LogbackHelper.java
  2. 15
    4
      server/sonar-process/src/test/java/org/sonar/process/LogbackHelperTest.java
  3. 1
    2
      server/sonar-search/src/main/java/org/sonar/search/SearchLogging.java
  4. 8
    6
      server/sonar-server/src/main/java/org/sonar/server/app/WebLogging.java
  5. 45
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/CePropertyDefinitions.java
  6. 7
    3
      server/sonar-server/src/main/java/org/sonar/server/computation/CeWorkerImpl.java
  7. 67
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/log/CeFileAppenderFactory.java
  8. 36
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogAcceptFilter.java
  9. 36
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogDenyFilter.java
  10. 164
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogging.java
  11. 105
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/log/LogFileRef.java
  12. 23
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/log/package-info.java
  13. 1
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/ws/CeWsModule.java
  14. 114
    0
      server/sonar-server/src/main/java/org/sonar/server/computation/ws/LogsWsAction.java
  15. 11
    1
      server/sonar-server/src/main/java/org/sonar/server/computation/ws/TaskFormatter.java
  16. 0
    1
      server/sonar-server/src/main/java/org/sonar/server/computation/ws/TaskWsAction.java
  17. 2
    0
      server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel1.java
  18. 4
    2
      server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java
  19. 4
    2
      server/sonar-server/src/main/resources/org/sonar/server/computation/ws/activity-example.json
  20. 4
    2
      server/sonar-server/src/main/resources/org/sonar/server/computation/ws/project-example.json
  21. 4
    2
      server/sonar-server/src/main/resources/org/sonar/server/computation/ws/queue-example.json
  22. 1
    2
      server/sonar-server/src/test/java/org/sonar/server/app/ProgrammaticLogbackValveTest.java
  23. 20
    3
      server/sonar-server/src/test/java/org/sonar/server/app/WebLoggingTest.java
  24. 38
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/CePropertyDefinitionsTest.java
  25. 16
    7
      server/sonar-server/src/test/java/org/sonar/server/computation/CeWorkerImplTest.java
  26. 46
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/log/CeFileAppenderFactoryTest.java
  27. 52
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/log/CeLogAcceptFilterTest.java
  28. 52
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/log/CeLogDenyFilterTest.java
  29. 154
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/log/CeLoggingTest.java
  30. 84
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/log/LogFileRefTest.java
  31. 17
    1
      server/sonar-server/src/test/java/org/sonar/server/computation/ws/ActivityWsActionTest.java
  32. 1
    1
      server/sonar-server/src/test/java/org/sonar/server/computation/ws/CeWsModuleTest.java
  33. 136
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/ws/LogsWsActionTest.java
  34. 15
    1
      server/sonar-server/src/test/java/org/sonar/server/computation/ws/ProjectWsActionTest.java
  35. 15
    1
      server/sonar-server/src/test/java/org/sonar/server/computation/ws/QueueWsActionTest.java
  36. 191
    0
      server/sonar-server/src/test/java/org/sonar/server/computation/ws/TaskFormatterTest.java
  37. 17
    1
      server/sonar-server/src/test/java/org/sonar/server/computation/ws/TaskWsActionTest.java
  38. 2
    3
      sonar-application/src/main/java/org/sonar/application/AppLogging.java
  39. 0
    3
      sonar-core/src/main/java/org/sonar/core/config/PurgeConstants.java
  40. 92
    5
      sonar-ws/src/main/gen-java/org/sonarqube/ws/WsCe.java
  41. 1
    0
      sonar-ws/src/main/protobuf/ws-ce.proto

+ 7
- 3
server/sonar-process/src/main/java/org/sonar/process/LogbackHelper.java Näytä tiedosto

@@ -29,16 +29,17 @@ import ch.qos.logback.classic.spi.LoggerContextListener;
import ch.qos.logback.core.ConsoleAppender;
import ch.qos.logback.core.Context;
import ch.qos.logback.core.FileAppender;
import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.joran.spi.JoranException;
import ch.qos.logback.core.rolling.FixedWindowRollingPolicy;
import ch.qos.logback.core.rolling.RollingFileAppender;
import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy;
import ch.qos.logback.core.rolling.TimeBasedRollingPolicy;
import java.io.File;
import javax.annotation.Nullable;
import org.apache.commons.lang.StringUtils;
import org.slf4j.LoggerFactory;

import java.io.File;

/**
* Helps to configure Logback in a programmatic way, without using XML.
*/
@@ -72,7 +73,7 @@ public class LogbackHelper {
return propagator;
}

public ConsoleAppender newConsoleAppender(Context loggerContext, String name, String pattern) {
public ConsoleAppender newConsoleAppender(Context loggerContext, String name, String pattern, @Nullable Filter filter) {
PatternLayoutEncoder consoleEncoder = new PatternLayoutEncoder();
consoleEncoder.setContext(loggerContext);
consoleEncoder.setPattern(pattern);
@@ -82,6 +83,9 @@ public class LogbackHelper {
consoleAppender.setEncoder(consoleEncoder);
consoleAppender.setName(name);
consoleAppender.setTarget("System.out");
if (filter != null) {
consoleAppender.addFilter(filter);
}
consoleAppender.start();
return consoleAppender;
}

+ 15
- 4
server/sonar-process/src/test/java/org/sonar/process/LogbackHelperTest.java Näytä tiedosto

@@ -27,21 +27,22 @@ import ch.qos.logback.classic.spi.LoggerContextListener;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.ConsoleAppender;
import ch.qos.logback.core.FileAppender;
import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.rolling.FixedWindowRollingPolicy;
import ch.qos.logback.core.rolling.RollingFileAppender;
import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy;
import ch.qos.logback.core.rolling.TimeBasedRollingPolicy;
import java.io.File;
import java.util.Properties;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.File;
import java.util.Properties;

import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;

public class LogbackHelperTest {

@@ -81,12 +82,22 @@ public class LogbackHelperTest {
@Test
public void newConsoleAppender() {
LoggerContext ctx = underTest.getRootContext();
ConsoleAppender<?> appender = underTest.newConsoleAppender(ctx, "MY_APPENDER", "%msg%n");
ConsoleAppender<?> appender = underTest.newConsoleAppender(ctx, "MY_APPENDER", "%msg%n", null);

assertThat(appender.getName()).isEqualTo("MY_APPENDER");
assertThat(appender.getContext()).isSameAs(ctx);
assertThat(appender.isStarted()).isTrue();
assertThat(((PatternLayoutEncoder) appender.getEncoder()).getPattern()).isEqualTo("%msg%n");
assertThat(appender.getCopyOfAttachedFiltersList()).isEmpty();
}

@Test
public void newConsoleAppender_with_filter() {
Filter filter = mock(Filter.class);
LoggerContext ctx = underTest.getRootContext();
ConsoleAppender<?> appender = underTest.newConsoleAppender(ctx, "MY_APPENDER", "%msg%n", filter);

assertThat(appender.getCopyOfAttachedFiltersList()).containsOnly(filter);
}

@Test

+ 1
- 2
server/sonar-search/src/main/java/org/sonar/search/SearchLogging.java Näytä tiedosto

@@ -24,7 +24,6 @@ import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.ConsoleAppender;

import org.sonar.process.LogbackHelper;

public class SearchLogging {
@@ -37,7 +36,7 @@ public class SearchLogging {
LoggerContext ctx = helper.getRootContext();
ctx.reset();

ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(ctx, "CONSOLE", LOG_FORMAT);
ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(ctx, "CONSOLE", LOG_FORMAT, null);
Logger rootLogger = helper.configureLogger(ctx, Logger.ROOT_LOGGER_NAME, Level.INFO);
rootLogger.addAppender(consoleAppender);
return ctx;

+ 8
- 6
server/sonar-server/src/main/java/org/sonar/server/app/WebLogging.java Näytä tiedosto

@@ -24,13 +24,13 @@ import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.ConsoleAppender;
import java.util.logging.LogManager;
import org.slf4j.bridge.SLF4JBridgeHandler;
import org.sonar.api.utils.MessageException;
import org.sonar.process.LogbackHelper;
import org.sonar.process.Props;
import java.util.logging.LogManager;
import org.sonar.server.computation.log.CeLogDenyFilter;
import org.sonar.server.computation.log.CeLogging;

/**
* Configure logback for web server process. Logs must be written to console, which is
@@ -48,7 +48,7 @@ class WebLogging {
ctx.reset();

helper.enableJulChangePropagation(ctx);
configureAppender(ctx);
configureAppender(ctx, props);
configureLevels(ctx, props);

// Configure java.util.logging, used by Tomcat, in order to forward to slf4j
@@ -57,9 +57,11 @@ class WebLogging {
return ctx;
}

private void configureAppender(LoggerContext ctx) {
ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(ctx, "CONSOLE", LOG_FORMAT);
private void configureAppender(LoggerContext ctx, Props props) {
ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(ctx, "CONSOLE", LOG_FORMAT, new CeLogDenyFilter<ILoggingEvent>());
ctx.getLogger(Logger.ROOT_LOGGER_NAME).addAppender(consoleAppender);
ctx.getLogger(Logger.ROOT_LOGGER_NAME).addAppender(CeLogging.createAppenderConfiguration(ctx, props));

}

private void configureLevels(LoggerContext ctx, Props props) {

+ 45
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/CePropertyDefinitions.java Näytä tiedosto

@@ -0,0 +1,45 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation;

import java.util.List;
import org.sonar.api.CoreProperties;
import org.sonar.api.PropertyType;
import org.sonar.api.config.PropertyDefinition;
import org.sonar.server.computation.log.CeLogging;

import static java.util.Arrays.asList;

public class CePropertyDefinitions {
private CePropertyDefinitions() {
// only statics
}

public static List<PropertyDefinition> all() {
return asList(
PropertyDefinition.builder(CeLogging.MAX_LOGS_PROPERTY)
.name("Compute Engine Log Retention")
.description("Number of tasks to keep logs for a given project. Once the number of logs exceeds this limit, oldest logs are purged.")
.type(PropertyType.INTEGER)
.defaultValue("10")
.category(CoreProperties.CATEGORY_GENERAL)
.build());
}
}

+ 7
- 3
server/sonar-server/src/main/java/org/sonar/server/computation/CeWorkerImpl.java Näytä tiedosto

@@ -25,6 +25,7 @@ import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.core.util.logs.Profiler;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.server.computation.log.CeLogging;

import static java.lang.String.format;

@@ -34,10 +35,12 @@ public class CeWorkerImpl implements CeWorker {

private final CeQueue queue;
private final ReportTaskProcessor reportTaskProcessor;
private final CeLogging ceLogging;

public CeWorkerImpl(CeQueue queue, ReportTaskProcessor reportTaskProcessor) {
public CeWorkerImpl(CeQueue queue, ReportTaskProcessor reportTaskProcessor, CeLogging ceLogging) {
this.queue = queue;
this.reportTaskProcessor = reportTaskProcessor;
this.ceLogging = ceLogging;
}

@Override
@@ -54,17 +57,18 @@ public class CeWorkerImpl implements CeWorker {
return;
}

// TODO delegate the message to the related task processor, according to task type
ceLogging.initForTask(task);
Profiler profiler = Profiler.create(LOG).startInfo(format("Analysis of project %s (report %s)", task.getComponentKey(), task.getUuid()));
try {
// TODO delegate the message to the related task processor, according to task type
reportTaskProcessor.process(task);
queue.remove(task, CeActivityDto.Status.SUCCESS);
} catch (Throwable e) {
LOG.error(format("Failed to process task %s", task.getUuid()), e);
queue.remove(task, CeActivityDto.Status.FAILED);
} finally {

profiler.stopInfo();
ceLogging.clearForTask();
}
}
}

+ 67
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/log/CeFileAppenderFactory.java Näytä tiedosto

@@ -0,0 +1,67 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
import ch.qos.logback.core.Context;
import ch.qos.logback.core.FileAppender;
import ch.qos.logback.core.sift.AppenderFactory;
import com.google.common.annotations.VisibleForTesting;
import java.io.File;
import org.sonar.server.computation.CeTask;

import static java.lang.String.format;

/**
* Creates a Logback appender for a Compute Engine task. See
* http://logback.qos.ch/manual/loggingSeparation.html
*/
public class CeFileAppenderFactory<E> implements AppenderFactory<E> {

private static final String ENCODER_PATTERN = "%d{yyyy.MM.dd HH:mm:ss} %-5level [%logger{20}] %msg%n";

private final File ceLogsDir;

@VisibleForTesting
CeFileAppenderFactory(File ceLogsDir) {
this.ceLogsDir = ceLogsDir;
}

/**
* @param context
* @param discriminatingValue path of log file relative to the directory data/ce/logs
* @see CeLogging#initForTask(CeTask)
*/
@Override
public FileAppender<E> buildAppender(Context context, String discriminatingValue) {
PatternLayoutEncoder consoleEncoder = new PatternLayoutEncoder();
consoleEncoder.setContext(context);
consoleEncoder.setPattern(ENCODER_PATTERN);
consoleEncoder.start();
FileAppender appender = new FileAppender<>();
appender.setContext(context);
appender.setEncoder(consoleEncoder);
appender.setName(format("ce-%s", discriminatingValue));
appender.setFile(new File(ceLogsDir, discriminatingValue).getAbsolutePath());
appender.start();
return appender;
}

}

+ 36
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogAcceptFilter.java Näytä tiedosto

@@ -0,0 +1,36 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply;
import org.slf4j.MDC;

/**
* Keeps only the Compute Engine logs.
*/
public class CeLogAcceptFilter<E> extends Filter<E> {

@Override
public FilterReply decide(E o) {
return MDC.get(CeLogging.MDC_LOG_PATH) == null ? FilterReply.DENY : FilterReply.ACCEPT;
}

}

+ 36
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogDenyFilter.java Näytä tiedosto

@@ -0,0 +1,36 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply;
import org.slf4j.MDC;

/**
* Filters out the Compute Engine logs.
*/
public class CeLogDenyFilter<E> extends Filter<E> {

@Override
public FilterReply decide(E o) {
return MDC.get(CeLogging.MDC_LOG_PATH) == null ? FilterReply.ACCEPT : FilterReply.DENY;
}

}

+ 164
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/log/CeLogging.java Näytä tiedosto

@@ -0,0 +1,164 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.sift.MDCBasedDiscriminator;
import ch.qos.logback.classic.sift.SiftingAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import java.io.File;
import java.util.Collections;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.comparator.LastModifiedFileComparator;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.log4j.MDC;
import org.sonar.api.config.Settings;
import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;
import org.sonar.server.computation.CeTask;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.FluentIterable.from;
import static com.google.common.collect.Lists.newArrayList;
import static java.lang.String.format;

/**
* Manages the logs written by Compute Engine:
* <ul>
* <li>access to existing logs</li>
* <li>configure logback when CE worker starts and stops processing a task</li>
* </ul>
*/
public class CeLogging {

@VisibleForTesting
static final String MDC_LOG_PATH = "ceLogPath";
public static final String MAX_LOGS_PROPERTY = "sonar.ce.maxLogsPerTask";

private final File logsDir;
private final int maxLogs;

public CeLogging(Settings settings) {
String dataDir = settings.getString(ProcessProperties.PATH_DATA);
checkArgument(dataDir != null, "Property %s is not set", ProcessProperties.PATH_DATA);
this.logsDir = logsDirFromDataDir(new File(dataDir));
this.maxLogs = settings.getInt(MAX_LOGS_PROPERTY);
if (maxLogs < 0) {
throw new IllegalArgumentException(format("Property %s must be positive. Got: %d", MAX_LOGS_PROPERTY, maxLogs));
}
}

@VisibleForTesting
CeLogging(File logsDir, int maxLogs) {
this.logsDir = logsDir;
this.maxLogs = maxLogs;
}

/**
* Gets the log file of a given task. It may not exist if it
* was purged or if the task does not exist.
*/
public Optional<File> getFile(LogFileRef ref) {
File logFile = new File(logsDir, ref.getRelativePath());
if (logFile.exists()) {
return Optional.of(logFile);
}
return Optional.absent();
}

/**
* Initialize logging of a Compute Engine task. Must be called
* before first writing of log.
*/
public void initForTask(CeTask task) {
LogFileRef ref = LogFileRef.from(task);
// Logback SiftingAppender requires to use a String, so
// the path is put but not the object LogFileRef
MDC.put(MDC_LOG_PATH, ref.getRelativePath());
}

/**
* Clean-up the logging of a task. Must be called after the last writing
* of log.
*/
public void clearForTask() {
String relativePath = (String) MDC.get(MDC_LOG_PATH);
MDC.remove(MDC_LOG_PATH);

if (relativePath != null) {
purgeDir(new File(logsDir, relativePath).getParentFile());
}
}

@VisibleForTesting
void purgeDir(File dir) {
if (dir.exists()) {
List<File> logFiles = newArrayList(FileUtils.listFiles(dir, FileFilterUtils.fileFileFilter(), FileFilterUtils.falseFileFilter()));
if (logFiles.size() > maxLogs) {
Collections.sort(logFiles, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR);
for (File logFile : from(logFiles).limit(logFiles.size() - maxLogs)) {
logFile.delete();
}
}
}
}

/**
* Directory which contains all the compute engine logs.
* Log files must be persistent among server restarts and upgrades, so they are
* stored into directory data/ but not into directories logs/ or temp/.
* @return the non-null directory. It may not exist at startup.
*/
static File logsDirFromDataDir(File dataDir) {
return new File(dataDir, "ce/logs");
}

/**
* Create Logback configuration for enabling sift appender.
* A new log file is created for each task. It is based on MDC as long
* as Compute Engine is not executed in its
* own process but in the same process as web server.
*/
public static Appender<ILoggingEvent> createAppenderConfiguration(LoggerContext ctx, Props processProps) {
File dataDir = new File(processProps.nonNullValue(ProcessProperties.PATH_DATA));
File logsDir = logsDirFromDataDir(dataDir);
return createAppenderConfiguration(ctx, logsDir);
}

static SiftingAppender createAppenderConfiguration(LoggerContext ctx, File logsDir) {
SiftingAppender siftingAppender = new SiftingAppender();
siftingAppender.addFilter(new CeLogAcceptFilter<ILoggingEvent>());
MDCBasedDiscriminator mdcDiscriminator = new MDCBasedDiscriminator();
mdcDiscriminator.setContext(ctx);
mdcDiscriminator.setKey(MDC_LOG_PATH);
mdcDiscriminator.setDefaultValue("error");
mdcDiscriminator.start();
siftingAppender.setContext(ctx);
siftingAppender.setDiscriminator(mdcDiscriminator);
siftingAppender.setAppenderFactory(new CeFileAppenderFactory(logsDir));
siftingAppender.setName("ce");
siftingAppender.start();
return siftingAppender;
}
}

+ 105
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/log/LogFileRef.java Näytä tiedosto

@@ -0,0 +1,105 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import com.google.common.annotations.VisibleForTesting;
import java.util.regex.Pattern;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.server.computation.CeTask;

import static java.lang.String.format;

public class LogFileRef {

// restricted white-list for now
private static final Pattern FILENAME_PATTERN = Pattern.compile("^[\\w\\-]*$");
private final String taskType;
private final String taskUuid;

@CheckForNull
private final String componentUuid;

public LogFileRef(String taskType, String taskUuid, @Nullable String componentUuid) {
this.taskType = requireValidFilename(taskType);
this.taskUuid = requireValidFilename(taskUuid);
this.componentUuid = requireValidFilename(componentUuid);
}

@VisibleForTesting
@CheckForNull
static String requireValidFilename(@Nullable String s) {
if (s != null && !FILENAME_PATTERN.matcher(s).matches()) {
throw new IllegalArgumentException(String.format("'%s' is not a valid filename for Compute Engine logs", s));
}
return s;
}

/**
* Path relative to the CE logs directory
*/
public String getRelativePath() {
if (componentUuid == null) {
return format("%s/%s.log", taskType, taskUuid);
}
return format("%s/%s/%s.log", taskType, componentUuid, taskUuid);
}

@Override
public boolean equals(@Nullable Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LogFileRef that = (LogFileRef) o;
if (!taskType.equals(that.taskType)) {
return false;
}
if (!taskUuid.equals(that.taskUuid)) {
return false;
}
return !(componentUuid != null ? !componentUuid.equals(that.componentUuid) : that.componentUuid != null);

}

@Override
public int hashCode() {
int result = taskType.hashCode();
result = 31 * result + taskUuid.hashCode();
result = 31 * result + (componentUuid != null ? componentUuid.hashCode() : 0);
return result;
}

public static LogFileRef from(CeActivityDto dto) {
return new LogFileRef(dto.getTaskType(), dto.getUuid(), dto.getComponentUuid());
}

public static LogFileRef from(CeQueueDto dto) {
return new LogFileRef(dto.getTaskType(), dto.getUuid(), dto.getComponentUuid());
}

public static LogFileRef from(CeTask task) {
return new LogFileRef(task.getType(), task.getUuid(), task.getComponentUuid());
}
}

+ 23
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/log/package-info.java Näytä tiedosto

@@ -0,0 +1,23 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
@ParametersAreNonnullByDefault
package org.sonar.server.computation.log;

import javax.annotation.ParametersAreNonnullByDefault;

+ 1
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/ws/CeWsModule.java Näytä tiedosto

@@ -30,6 +30,7 @@ public class CeWsModule extends Module {
CeQueueWsAction.class,
CeWs.class,
IsQueueEmptyWs.class,
LogsWsAction.class,
ProjectWsAction.class,
SubmitWsAction.class,
TaskFormatter.class,

+ 114
- 0
server/sonar-server/src/main/java/org/sonar/server/computation/ws/LogsWsAction.java Näytä tiedosto

@@ -0,0 +1,114 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.ws;

import com.google.common.base.Optional;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.sonar.api.server.ws.Request;
import org.sonar.api.server.ws.Response;
import org.sonar.api.server.ws.WebService;
import org.sonar.api.web.UserRole;
import org.sonar.core.util.Uuids;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonar.server.exceptions.NotFoundException;
import org.sonar.server.plugins.MimeTypes;
import org.sonar.server.user.UserSession;

import static java.lang.String.format;

public class LogsWsAction implements CeWsAction {

public static final String ACTION = "logs";
public static final String PARAM_TASK_UUID = "taskId";

private final DbClient dbClient;
private final UserSession userSession;
private final CeLogging ceLogging;

public LogsWsAction(DbClient dbClient, UserSession userSession, CeLogging ceLogging) {
this.dbClient = dbClient;
this.userSession = userSession;
this.ceLogging = ceLogging;
}

@Override
public void define(WebService.NewController controller) {
WebService.NewAction action = controller.createAction(ACTION)
.setDescription("Logs of a task. Returns HTTP code 404 if task does not " +
"exist or if logs are not available. Requires system administration permission.")
.setInternal(true)
.setHandler(this);

action
.createParam(PARAM_TASK_UUID)
.setRequired(true)
.setDescription("Id of task")
.setExampleValue(Uuids.UUID_EXAMPLE_01);
}

@Override
public void handle(Request wsRequest, Response wsResponse) throws Exception {
userSession.checkGlobalPermission(UserRole.ADMIN);

String taskUuid = wsRequest.mandatoryParam(PARAM_TASK_UUID);
LogFileRef ref = loadLogRef(taskUuid);
Optional<File> logFile = ceLogging.getFile(ref);
if (logFile.isPresent()) {
writeFile(logFile.get(), wsResponse);
} else {
throw new NotFoundException(format("Logs of task %s not found", taskUuid));
}
}

private LogFileRef loadLogRef(String taskUuid) {
DbSession dbSession = dbClient.openSession(false);
try {
Optional<CeQueueDto> queueDto = dbClient.ceQueueDao().selectByUuid(dbSession, taskUuid);
if (queueDto.isPresent()) {
return LogFileRef.from(queueDto.get());
}
Optional<CeActivityDto> activityDto = dbClient.ceActivityDao().selectByUuid(dbSession, taskUuid);
if (activityDto.isPresent()) {
return LogFileRef.from(activityDto.get());
}
throw new NotFoundException(format("Task %s not found", taskUuid));

} finally {
dbClient.closeSession(dbSession);
}
}

private static void writeFile(File file, Response wsResponse) {
try {
Response.Stream stream = wsResponse.stream();
stream.setMediaType(MimeTypes.TXT);
FileUtils.copyFile(file, stream.output());
} catch (IOException e) {
throw new IllegalStateException("Fail to copy compute engine log file to HTTP response: " + file.getAbsolutePath(), e);
}
}
}

+ 11
- 1
server/sonar-server/src/main/java/org/sonar/server/computation/ws/TaskFormatter.java Näytä tiedosto

@@ -33,14 +33,22 @@ import org.sonar.db.DbSession;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.component.ComponentDto;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonarqube.ws.WsCe;

/**
* Converts {@link CeActivityDto} and {@link CeQueueDto} to the protobuf objects
* used to write WS responses (see ws-ce.proto in module sonar-ws)
*/
public class TaskFormatter {

private final DbClient dbClient;
private final CeLogging ceLogging;

public TaskFormatter(DbClient dbClient) {
public TaskFormatter(DbClient dbClient, CeLogging ceLogging) {
this.dbClient = dbClient;
this.ceLogging = ceLogging;
}

public List<WsCe.Task> formatQueue(DbSession dbSession, List<CeQueueDto> dtos) {
@@ -61,6 +69,7 @@ public class TaskFormatter {
builder.setId(dto.getUuid());
builder.setStatus(WsCe.TaskStatus.valueOf(dto.getStatus().name()));
builder.setType(dto.getTaskType());
builder.setLogs(ceLogging.getFile(LogFileRef.from(dto)).isPresent());
if (dto.getComponentUuid() != null) {
builder.setComponentId(dto.getComponentUuid());
buildComponent(builder, componentCache.get(dto.getComponentUuid()));
@@ -93,6 +102,7 @@ public class TaskFormatter {
builder.setId(dto.getUuid());
builder.setStatus(WsCe.TaskStatus.valueOf(dto.getStatus().name()));
builder.setType(dto.getTaskType());
builder.setLogs(ceLogging.getFile(LogFileRef.from(dto)).isPresent());
if (dto.getComponentUuid() != null) {
builder.setComponentId(dto.getComponentUuid());
buildComponent(builder, componentCache.get(dto.getComponentUuid()));

+ 0
- 1
server/sonar-server/src/main/java/org/sonar/server/computation/ws/TaskWsAction.java Näytä tiedosto

@@ -87,6 +87,5 @@ public class TaskWsAction implements CeWsAction {
} finally {
dbClient.closeSession(dbSession);
}

}
}

+ 2
- 0
server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel1.java Näytä tiedosto

@@ -33,6 +33,7 @@ import org.sonar.db.purge.PurgeProfiler;
import org.sonar.db.semaphore.SemaphoresImpl;
import org.sonar.db.version.DatabaseVersion;
import org.sonar.db.version.MigrationStepModule;
import org.sonar.server.computation.CePropertyDefinitions;
import org.sonar.server.db.DbClient;
import org.sonar.server.db.EmbeddedDatabaseFactory;
import org.sonar.server.issue.index.IssueIndex;
@@ -118,6 +119,7 @@ public class PlatformLevel1 extends PlatformLevel {
org.sonar.core.properties.PropertiesDao.class,
org.sonar.core.persistence.MyBatis.class);
addAll(CorePropertyDefinitions.all());
addAll(CePropertyDefinitions.all());
add(MigrationStepModule.class);
}


+ 4
- 2
server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java Näytä tiedosto

@@ -31,8 +31,6 @@ import org.sonar.api.rules.AnnotationRuleParser;
import org.sonar.api.rules.XMLRuleParser;
import org.sonar.api.server.rule.RulesDefinitionXmlLoader;
import org.sonar.core.component.DefaultResourceTypes;
import org.sonar.server.computation.dbcleaner.IndexPurgeListener;
import org.sonar.server.computation.dbcleaner.ProjectCleaner;
import org.sonar.core.issue.IssueUpdater;
import org.sonar.core.issue.workflow.FunctionExecutor;
import org.sonar.core.issue.workflow.IssueWorkflow;
@@ -71,6 +69,9 @@ import org.sonar.server.computation.CleanReportQueueListener;
import org.sonar.server.computation.ComputeEngineProcessingModule;
import org.sonar.server.computation.ReportFiles;
import org.sonar.server.computation.ReportSubmitter;
import org.sonar.server.computation.dbcleaner.IndexPurgeListener;
import org.sonar.server.computation.dbcleaner.ProjectCleaner;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.monitoring.CEQueueStatusImpl;
import org.sonar.server.computation.monitoring.ComputeEngineQueueMonitor;
import org.sonar.server.computation.ws.CeWsModule;
@@ -718,6 +719,7 @@ public class PlatformLevel4 extends PlatformLevel {
ReportFiles.class,
ComputeEngineProcessingModule.class,
CeWsModule.class,
CeLogging.class,
DefaultPeriodCleaner.class,
ProjectCleaner.class,
ProjectSettingsFactory.class,

+ 4
- 2
server/sonar-server/src/main/resources/org/sonar/server/computation/ws/activity-example.json Näytä tiedosto

@@ -16,7 +16,8 @@
"submitterLogin": "john",
"startedAt": "2015-08-13T23:35:00+0200",
"finishedAt": "2015-08-13T23:35:10+0200",
"executionTimeMs": 10000
"executionTimeMs": 10000,
"logs": true
},
{
"id": "AU_dO1vsORa8_beWCwmP",
@@ -28,7 +29,8 @@
"submittedAt": "2015-09-17T23:34:59+0200",
"startedAt": "2015-09-17T23:35:00+0200",
"finishedAt": "2015-08-13T23:37:00+0200",
"executionTimeMs": 120000
"executionTimeMs": 120000,
"logs": false
}
]
}

+ 4
- 2
server/sonar-server/src/main/resources/org/sonar/server/computation/ws/project-example.json Näytä tiedosto

@@ -7,7 +7,8 @@
"componentKey": "com.github.kevinsawicki:http-request-parent",
"componentName": "HttpRequest",
"status": "PENDING",
"submittedAt": "2015-09-21T19:28:54+0200"
"submittedAt": "2015-09-21T19:28:54+0200",
"logs": true
}
],
"current": {
@@ -20,6 +21,7 @@
"submittedAt": "2015-09-21T19:25:49+0200",
"startedAt": "2015-09-21T19:25:57+0200",
"finishedAt": "2015-09-21T19:25:58+0200",
"executionTimeMs": 1371
"executionTimeMs": 1371,
"logs": true
}
}

+ 4
- 2
server/sonar-server/src/main/resources/org/sonar/server/computation/ws/queue-example.json Näytä tiedosto

@@ -8,7 +8,8 @@
"componentName": "Project One",
"status": "IN_PROGRESS",
"submittedAt": "2015-08-13T23:34:59+0200",
"submitterLogin": "john"
"submitterLogin": "john",
"logs": true
},
{
"id": "AU_dO1vsORa8_beWCwmP",
@@ -18,7 +19,8 @@
"componentName": "Project Two",
"status": "PENDING",
"submittedAt": "2015-09-17T23:34:59+0200",
"startedAt": "2015-09-17T23:35:00+0200"
"startedAt": "2015-09-17T23:35:00+0200",
"logs": true
}
]
}

+ 1
- 2
server/sonar-server/src/test/java/org/sonar/server/app/ProgrammaticLogbackValveTest.java Näytä tiedosto

@@ -21,7 +21,6 @@ package org.sonar.server.app;

import ch.qos.logback.access.spi.IAccessEvent;
import ch.qos.logback.core.ConsoleAppender;

import org.apache.catalina.Container;
import org.junit.AfterClass;
import org.junit.Test;
@@ -42,7 +41,7 @@ public class ProgrammaticLogbackValveTest {
ProgrammaticLogbackValve valve = new ProgrammaticLogbackValve();
valve.setContainer(mock(Container.class));
LogbackHelper helper = new LogbackHelper();
ConsoleAppender<IAccessEvent> appender = helper.newConsoleAppender(valve, "CONSOLE", "combined");
ConsoleAppender<IAccessEvent> appender = helper.newConsoleAppender(valve, "CONSOLE", "combined", null);
valve.addAppender(appender);

valve.start();

+ 20
- 3
server/sonar-server/src/test/java/org/sonar/server/app/WebLoggingTest.java Näytä tiedosto

@@ -24,22 +24,39 @@ import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.ConsoleAppender;
import ch.qos.logback.core.joran.spi.JoranException;
import java.io.IOException;
import java.util.Properties;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.process.LogbackHelper;
import org.sonar.process.ProcessProperties;
import org.sonar.process.Props;

import java.util.Properties;

import static org.assertj.core.api.Assertions.assertThat;

public class WebLoggingTest {

@Rule
public TemporaryFolder temp = new TemporaryFolder();

Props props = new Props(new Properties());
WebLogging underTest = new WebLogging();

/**
* Path to data dir must be set for Compute Engine logging.
* @see org.sonar.server.computation.log.CeFileAppenderFactory
*/
@Before
public void setUp() throws IOException {
props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
}

@AfterClass
public static void resetLogback() throws Exception {
public static void resetLogback() throws JoranException {
new LogbackHelper().resetFromXml("/logback-test.xml");
}


+ 38
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/CePropertyDefinitionsTest.java Näytä tiedosto

@@ -0,0 +1,38 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation;

import org.junit.Test;
import org.sonar.test.TestUtils;

import static org.assertj.core.api.Assertions.assertThat;

public class CePropertyDefinitionsTest {

@Test
public void all() {
assertThat(CePropertyDefinitions.all()).isNotEmpty();
}

@Test
public void only_statics() {
assertThat(TestUtils.hasOnlyPrivateConstructors(CePropertyDefinitions.class));
}
}

+ 16
- 7
server/sonar-server/src/test/java/org/sonar/server/computation/CeWorkerImplTest.java Näytä tiedosto

@@ -21,12 +21,14 @@ package org.sonar.server.computation;

import com.google.common.base.Optional;
import org.junit.Test;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.server.computation.log.CeLogging;

import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;

@@ -34,7 +36,8 @@ public class CeWorkerImplTest {

CeQueue queue = mock(CeQueueImpl.class);
ReportTaskProcessor taskProcessor = mock(ReportTaskProcessor.class);
CeWorker underTest = new CeWorkerImpl(queue, taskProcessor);
CeLogging ceLogging = mock(CeLogging.class);
CeWorker underTest = new CeWorkerImpl(queue, taskProcessor, ceLogging);

@Test
public void no_pending_tasks_in_queue() throws Exception {
@@ -42,7 +45,7 @@ public class CeWorkerImplTest {

underTest.run();

verifyZeroInteractions(taskProcessor);
verifyZeroInteractions(taskProcessor, ceLogging);
}

@Test
@@ -52,8 +55,11 @@ public class CeWorkerImplTest {

underTest.run();

verify(taskProcessor).process(task);
verify(queue).remove(task, CeActivityDto.Status.SUCCESS);
InOrder inOrder = Mockito.inOrder(ceLogging, taskProcessor, queue);
inOrder.verify(ceLogging).initForTask(task);
inOrder.verify(taskProcessor).process(task);
inOrder.verify(queue).remove(task, CeActivityDto.Status.SUCCESS);
inOrder.verify(ceLogging).clearForTask();
}

@Test
@@ -64,7 +70,10 @@ public class CeWorkerImplTest {

underTest.run();

verify(taskProcessor).process(task);
verify(queue).remove(task, CeActivityDto.Status.FAILED);
InOrder inOrder = Mockito.inOrder(ceLogging, taskProcessor, queue);
inOrder.verify(ceLogging).initForTask(task);
inOrder.verify(taskProcessor).process(task);
inOrder.verify(queue).remove(task, CeActivityDto.Status.FAILED);
inOrder.verify(ceLogging).clearForTask();
}
}

+ 46
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/log/CeFileAppenderFactoryTest.java Näytä tiedosto

@@ -0,0 +1,46 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.core.FileAppender;
import java.io.File;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import static org.assertj.core.api.Assertions.assertThat;

public class CeFileAppenderFactoryTest {

@Rule
public TemporaryFolder temp = new TemporaryFolder();

@Test
public void buildAppender() throws Exception {
File logsDir = temp.newFolder();
CeFileAppenderFactory factory = new CeFileAppenderFactory(logsDir);

FileAppender underTest = factory.buildAppender(new LoggerContext(), "uuid_1.log");

assertThat(new File(underTest.getFile())).isEqualTo(new File(logsDir, "uuid_1.log"));

}
}

+ 52
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/log/CeLogAcceptFilterTest.java Näytä tiedosto

@@ -0,0 +1,52 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply;
import org.apache.log4j.MDC;
import org.junit.After;
import org.junit.Test;

import static org.assertj.core.api.Assertions.assertThat;

public class CeLogAcceptFilterTest {

private static final Object UNUSED = "";

Filter underTest = new CeLogAcceptFilter();

@After
public void tearDown() {
MDC.clear();
}

@Test
public void reject_non_ce_logs() {
assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.DENY);
}

@Test
public void accept_ce_logs() {
MDC.put(CeLogging.MDC_LOG_PATH, "abc.log");
assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.ACCEPT);
}

}

+ 52
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/log/CeLogDenyFilterTest.java Näytä tiedosto

@@ -0,0 +1,52 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply;
import org.apache.log4j.MDC;
import org.junit.After;
import org.junit.Test;

import static org.assertj.core.api.Assertions.assertThat;

public class CeLogDenyFilterTest {

private static final Object UNUSED = "";

Filter underTest = new CeLogDenyFilter();

@After
public void tearDown() {
MDC.clear();
}

@Test
public void accept_non_ce_logs() {
assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.ACCEPT);
}

@Test
public void deny_ce_logs() {
MDC.put(CeLogging.MDC_LOG_PATH, "abc.log");
assertThat(underTest.decide(UNUSED)).isEqualTo(FilterReply.DENY);
}

}

+ 154
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/log/CeLoggingTest.java Näytä tiedosto

@@ -0,0 +1,154 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.sift.SiftingAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.filter.Filter;
import com.google.common.base.Optional;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.slf4j.MDC;
import org.sonar.api.config.Settings;
import org.sonar.process.ProcessProperties;
import org.sonar.server.computation.CeTask;

import static java.lang.String.format;
import static org.assertj.core.api.Assertions.assertThat;

public class CeLoggingTest {

@Rule
public ExpectedException expectedException = ExpectedException.none();

@Rule
public TemporaryFolder temp = new TemporaryFolder();

@Test
public void getFile() throws IOException {
File dataDir = temp.newFolder();
Settings settings = new Settings();
settings.setProperty(ProcessProperties.PATH_DATA, dataDir.getAbsolutePath());

CeLogging underTest = new CeLogging(settings);
LogFileRef ref = new LogFileRef("TYPE1", "TASK1", "COMPONENT1");

// file does not exist
Optional<File> file = underTest.getFile(ref);
assertThat(file.isPresent()).isFalse();

File logFile = new File(dataDir, "ce/logs/" + ref.getRelativePath());
FileUtils.touch(logFile);
file = underTest.getFile(ref);
assertThat(file.isPresent()).isTrue();
assertThat(file.get()).isEqualTo(logFile);
}

@Test(expected = IllegalArgumentException.class)
public void fail_if_data_dir_is_not_set() {
new CeLogging(new Settings());
}

@Test
public void use_MDC_to_store_path_to_in_progress_task_logs() throws IOException {
CeLogging underTest = new CeLogging(temp.newFolder(), 5);

CeTask task = new CeTask.Builder().setType("TYPE1").setUuid("U1").build();
underTest.initForTask(task);
assertThat(MDC.get(CeLogging.MDC_LOG_PATH)).isNotEmpty().isEqualTo(LogFileRef.from(task).getRelativePath());
underTest.clearForTask();
assertThat(MDC.get(CeLogging.MDC_LOG_PATH)).isNull();
}

@Test
public void delete_oldest_files_of_same_directory_to_keep_only_max_allowed_files() throws IOException {
File dir = temp.newFolder();
for (int i = 1; i <= 5; i++) {
File file = new File(dir, format("U%d.log", i));
FileUtils.touch(file);
// see javadoc: "all platforms support file-modification times to the nearest second,
// but some provide more precision" --> increment by second, not by millisecond
file.setLastModified(1_450_000_000_000L + i * 1000);
}
assertThat(dir.listFiles()).hasSize(5);

// keep 3 files in each dir
CeLogging underTest = new CeLogging(dir, 3);
underTest.purgeDir(dir);

assertThat(dir.listFiles()).hasSize(3);
assertThat(dir.listFiles()).extracting("name")
.containsOnly("U3.log", "U4.log", "U5.log");
}

@Test
public void do_not_delete_files_if_dir_has_less_files_than_max_allowed() throws IOException {
File dir = temp.newFolder();
FileUtils.touch(new File(dir, "U1.log"));

CeLogging underTest = new CeLogging(dir, 5);
underTest.purgeDir(dir);

assertThat(dir.listFiles()).extracting("name").containsOnly("U1.log");
}

@Test
public void do_not_keep_any_logs() throws IOException {
File dir = temp.newFolder();
FileUtils.touch(new File(dir, "U1.log"));

CeLogging underTest = new CeLogging(dir, 0);
underTest.purgeDir(dir);

assertThat(dir.listFiles()).isEmpty();
}

@Test
public void fail_if_max_logs_settings_is_negative() throws IOException {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Property sonar.ce.maxLogsPerTask must be positive. Got: -1");

Settings settings = new Settings();
settings.setProperty(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
settings.setProperty(CeLogging.MAX_LOGS_PROPERTY, -1);
new CeLogging(settings);
}

@Test
public void createConfiguration() throws Exception {
File logsDir = temp.newFolder();
SiftingAppender siftingAppender = CeLogging.createAppenderConfiguration(new LoggerContext(), logsDir);

// filter on CE logs
List<Filter<ILoggingEvent>> filters = siftingAppender.getCopyOfAttachedFiltersList();
assertThat(filters).hasSize(1);
assertThat(filters.get(0)).isInstanceOf(CeLogAcceptFilter.class);

assertThat(siftingAppender.getDiscriminator().getKey()).isEqualTo(CeLogging.MDC_LOG_PATH);
}

}

+ 84
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/log/LogFileRefTest.java Näytä tiedosto

@@ -0,0 +1,84 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.log;

import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.sonar.db.ce.CeTaskTypes;

import static org.assertj.core.api.Assertions.assertThat;

public class LogFileRefTest {

@Rule
public ExpectedException expectedException = ExpectedException.none();

@Test
public void equals_hashCode() {
LogFileRef ref1 = new LogFileRef(CeTaskTypes.REPORT, "UUID_1", "COMPONENT_1");
LogFileRef ref1bis = new LogFileRef(CeTaskTypes.REPORT, "UUID_1", "COMPONENT_1");
LogFileRef ref2 = new LogFileRef(CeTaskTypes.REPORT, "UUID_2", "COMPONENT_1");

assertThat(ref1.equals(ref1)).isTrue();
assertThat(ref1.equals(ref1bis)).isTrue();
assertThat(ref1.equals(ref2)).isFalse();
assertThat(ref1.equals(null)).isFalse();
assertThat(ref1.equals("UUID_1")).isFalse();

assertThat(ref1.hashCode()).isEqualTo(ref1bis.hashCode());
}

@Test
public void getRelativePath() {
assertThat(new LogFileRef("TYPE_1", "UUID_1", "COMPONENT_1").getRelativePath()).isEqualTo("TYPE_1/COMPONENT_1/UUID_1.log");
assertThat(new LogFileRef("TYPE_1", "UUID_1", null).getRelativePath()).isEqualTo("TYPE_1/UUID_1.log");
}

@Test
public void do_not_accept_invalid_task_type() {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("'foo/bar' is not a valid filename for Compute Engine logs");

new LogFileRef("foo/bar", "UUID", null);
}

@Test
public void do_not_accept_invalid_uuid() {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("'foo/bar' is not a valid filename for Compute Engine logs");

new LogFileRef("REPORT", "foo/bar", null);
}

@Test
public void do_not_accept_invalid_component_uuid() {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("'foo/bar' is not a valid filename for Compute Engine logs");

new LogFileRef("REPORT", "UUID", "foo/bar");
}

@Test
public void filename_must_support_uuid() {
String uuid = "AU-Tpxb-_iU5OvuD2FLy";
assertThat(LogFileRef.requireValidFilename(uuid)).isEqualTo(uuid);
}
}

+ 17
- 1
server/sonar-server/src/test/java/org/sonar/server/computation/ws/ActivityWsActionTest.java Näytä tiedosto

@@ -19,8 +19,11 @@
*/
package org.sonar.server.computation.ws;

import com.google.common.base.Optional;
import java.io.File;
import java.util.Collections;
import java.util.List;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.server.ws.WebService;
@@ -31,6 +34,8 @@ import org.sonar.db.DbTester;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonar.server.plugins.MimeTypes;
import org.sonar.server.tester.UserSessionRule;
import org.sonar.server.ws.TestResponse;
@@ -39,6 +44,9 @@ import org.sonarqube.ws.WsCe;

import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class ActivityWsActionTest {

@@ -48,10 +56,16 @@ public class ActivityWsActionTest {
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);

TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient());
CeLogging ceLogging = mock(CeLogging.class);
TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient(), ceLogging);
ActivityWsAction underTest = new ActivityWsAction(userSession, dbTester.getDbClient(), formatter);
WsActionTester tester = new WsActionTester(underTest);

@Before
public void setUp() {
when(ceLogging.getFile(any(LogFileRef.class))).thenReturn(Optional.<File>absent());
}

@Test
public void get_all_past_activity() {
userSession.setGlobalPermissions(UserRole.ADMIN);
@@ -71,9 +85,11 @@ public class ActivityWsActionTest {
assertThat(activityResponse.getTasks(0).getStatus()).isEqualTo(WsCe.TaskStatus.FAILED);
assertThat(activityResponse.getTasks(0).getComponentId()).isEqualTo("PROJECT_2");
assertThat(activityResponse.getTasks(0).getExecutionTimeMs()).isEqualTo(500L);
assertThat(activityResponse.getTasks(0).getLogs()).isFalse();
assertThat(activityResponse.getTasks(1).getId()).isEqualTo("T1");
assertThat(activityResponse.getTasks(1).getStatus()).isEqualTo(WsCe.TaskStatus.SUCCESS);
assertThat(activityResponse.getTasks(1).getComponentId()).isEqualTo("PROJECT_1");
assertThat(activityResponse.getTasks(1).getLogs()).isFalse();
}

@Test

+ 1
- 1
server/sonar-server/src/test/java/org/sonar/server/computation/ws/CeWsModuleTest.java Näytä tiedosto

@@ -30,6 +30,6 @@ public class CeWsModuleTest {
public void verify_count_of_added_components() {
ComponentContainer container = new ComponentContainer();
new CeWsModule().configure(container);
assertThat(container.size()).isEqualTo(9 + 2 /* injected by ComponentContainer */);
assertThat(container.size()).isEqualTo(10 + 2 /* injected by ComponentContainer */);
}
}

+ 136
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/ws/LogsWsActionTest.java Näytä tiedosto

@@ -0,0 +1,136 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.ws;

import com.google.common.base.Optional;
import java.io.File;
import java.io.IOException;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.api.utils.System2;
import org.sonar.api.web.UserRole;
import org.sonar.db.DbTester;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonar.server.exceptions.ForbiddenException;
import org.sonar.server.exceptions.NotFoundException;
import org.sonar.server.plugins.MimeTypes;
import org.sonar.server.tester.UserSessionRule;
import org.sonar.server.ws.TestResponse;
import org.sonar.server.ws.WsActionTester;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class LogsWsActionTest {

@Rule
public UserSessionRule userSession = UserSessionRule.standalone();

@Rule
public TemporaryFolder temp = new TemporaryFolder();

@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);

CeLogging ceLogging = mock(CeLogging.class);
LogsWsAction underTest = new LogsWsAction(dbTester.getDbClient(), userSession, ceLogging);
WsActionTester tester = new WsActionTester(underTest);

@Test
public void return_task_logs_if_available() throws IOException {
userSession.setGlobalPermissions(UserRole.ADMIN);

// task must exist in database
insert("TASK_1", null);
File logFile = temp.newFile();
FileUtils.write(logFile, "{logs}");
when(ceLogging.getFile(new LogFileRef(CeTaskTypes.REPORT, "TASK_1", null))).thenReturn(Optional.of(logFile));

TestResponse response = tester.newRequest()
.setParam("taskId", "TASK_1")
.execute();

assertThat(response.getMediaType()).isEqualTo(MimeTypes.TXT);
assertThat(response.getInput()).isEqualTo("{logs}");
}

/**
* The parameter taskId is present but empty. It's considered as
* a valid task which does not exist
*/
@Test(expected = NotFoundException.class)
public void return_404_if_task_id_is_empty() {
userSession.setGlobalPermissions(UserRole.ADMIN);
tester.newRequest()
.setParam("taskId", "")
.execute();
}

@Test(expected = IllegalArgumentException.class)
public void bad_request_if_task_id_is_missing() {
userSession.setGlobalPermissions(UserRole.ADMIN);
tester.newRequest()
.execute();
}

@Test(expected = NotFoundException.class)
public void return_404_if_task_logs_are_not_available() {
userSession.setGlobalPermissions(UserRole.ADMIN);
insert("TASK_1", null);
when(ceLogging.getFile(new LogFileRef(CeTaskTypes.REPORT, "TASK_1", null))).thenReturn(Optional.<File>absent());

tester.newRequest()
.setParam("taskId", "TASK_1")
.execute();
}

@Test(expected = NotFoundException.class)
public void return_404_if_task_does_not_exist() {
userSession.setGlobalPermissions(UserRole.ADMIN);
tester.newRequest()
.setParam("taskId", "TASK_1")
.execute();
}

@Test(expected = ForbiddenException.class)
public void require_admin_permission() {
tester.newRequest()
.setParam("taskId", "TASK_1")
.execute();
}

private CeQueueDto insert(String taskUuid, @Nullable String componentUuid) {
CeQueueDto queueDto = new CeQueueDto();
queueDto.setTaskType(CeTaskTypes.REPORT);
queueDto.setComponentUuid(componentUuid);
queueDto.setUuid(taskUuid);
queueDto.setStatus(CeQueueDto.Status.IN_PROGRESS);
dbTester.getDbClient().ceQueueDao().insert(dbTester.getSession(), queueDto);
dbTester.getSession().commit();
return queueDto;
}
}

+ 15
- 1
server/sonar-server/src/test/java/org/sonar/server/computation/ws/ProjectWsActionTest.java Näytä tiedosto

@@ -19,6 +19,9 @@
*/
package org.sonar.server.computation.ws;

import com.google.common.base.Optional;
import java.io.File;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.utils.System2;
@@ -28,6 +31,8 @@ import org.sonar.db.DbTester;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonar.server.plugins.MimeTypes;
import org.sonar.server.tester.UserSessionRule;
import org.sonar.server.ws.TestResponse;
@@ -35,6 +40,9 @@ import org.sonar.server.ws.WsActionTester;
import org.sonarqube.ws.WsCe;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class ProjectWsActionTest {

@@ -44,10 +52,16 @@ public class ProjectWsActionTest {
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);

TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient());
CeLogging ceLogging = mock(CeLogging.class);
TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient(), ceLogging);
ProjectWsAction underTest = new ProjectWsAction(userSession, dbTester.getDbClient(), formatter);
WsActionTester tester = new WsActionTester(underTest);

@Before
public void setUp() {
when(ceLogging.getFile(any(LogFileRef.class))).thenReturn(Optional.<File>absent());
}

@Test
public void empty_queue_and_empty_activity() {
userSession.addProjectUuidPermissions(UserRole.USER, "PROJECT_1");

+ 15
- 1
server/sonar-server/src/test/java/org/sonar/server/computation/ws/QueueWsActionTest.java Näytä tiedosto

@@ -19,6 +19,9 @@
*/
package org.sonar.server.computation.ws;

import com.google.common.base.Optional;
import java.io.File;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.utils.System2;
@@ -27,6 +30,8 @@ import org.sonar.core.util.Protobuf;
import org.sonar.db.DbTester;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonar.server.plugins.MimeTypes;
import org.sonar.server.tester.UserSessionRule;
import org.sonar.server.ws.TestResponse;
@@ -34,6 +39,9 @@ import org.sonar.server.ws.WsActionTester;
import org.sonarqube.ws.WsCe;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class QueueWsActionTest {

@@ -43,10 +51,16 @@ public class QueueWsActionTest {
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);

TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient());
CeLogging ceLogging = mock(CeLogging.class);
TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient(), ceLogging);
CeQueueWsAction underTest = new CeQueueWsAction(userSession, dbTester.getDbClient(), formatter);
WsActionTester tester = new WsActionTester(underTest);

@Before
public void setUp() {
when(ceLogging.getFile(any(LogFileRef.class))).thenReturn(Optional.<File>absent());
}

@Test
public void get_all_queue() {
userSession.setGlobalPermissions(UserRole.ADMIN);

+ 191
- 0
server/sonar-server/src/test/java/org/sonar/server/computation/ws/TaskFormatterTest.java Näytä tiedosto

@@ -0,0 +1,191 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.computation.ws;

import com.google.common.base.Optional;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import org.sonar.api.utils.DateUtils;
import org.sonar.api.utils.System2;
import org.sonar.db.DbTester;
import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.db.component.ComponentDto;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonarqube.ws.WsCe;

import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class TaskFormatterTest {

@Rule
public TemporaryFolder temp = new TemporaryFolder();

@Rule
public DbTester db = DbTester.create(System2.INSTANCE);

CeLogging ceLogging = mock(CeLogging.class, Mockito.RETURNS_DEEP_STUBS);
TaskFormatter underTest = new TaskFormatter(db.getDbClient(), ceLogging);

@Test
public void formatQueue_without_component() {
CeQueueDto dto = new CeQueueDto();
dto.setUuid("UUID");
dto.setTaskType("TYPE");
dto.setStatus(CeQueueDto.Status.PENDING);
dto.setCreatedAt(1_450_000_000_000L);

WsCe.Task wsTask = underTest.formatQueue(db.getSession(), dto);

assertThat(wsTask.getType()).isEqualTo("TYPE");
assertThat(wsTask.getId()).isEqualTo("UUID");
assertThat(wsTask.getStatus()).isEqualTo(WsCe.TaskStatus.PENDING);
assertThat(wsTask.getLogs()).isFalse();
assertThat(wsTask.getSubmittedAt()).isEqualTo(DateUtils.formatDateTime(new Date(1_450_000_000_000L)));

assertThat(wsTask.hasExecutionTimeMs()).isFalse();
assertThat(wsTask.hasSubmitterLogin()).isFalse();
assertThat(wsTask.hasComponentId()).isFalse();
assertThat(wsTask.hasComponentKey()).isFalse();
assertThat(wsTask.hasComponentName()).isFalse();
assertThat(wsTask.hasFinishedAt()).isFalse();
}

@Test
public void formatQueue_with_component_and_other_fields() throws IOException {
when(ceLogging.getFile(any(LogFileRef.class))).thenReturn(Optional.of(temp.newFile()));
db.getDbClient().componentDao().insert(db.getSession(), new ComponentDto().setUuid("COMPONENT_UUID").setKey("COMPONENT_KEY").setName("Component Name"));

CeQueueDto dto = new CeQueueDto();
dto.setUuid("UUID");
dto.setTaskType("TYPE");
dto.setStatus(CeQueueDto.Status.IN_PROGRESS);
dto.setCreatedAt(1_450_000_000_000L);
dto.setStartedAt(1_451_000_000_000L);
dto.setComponentUuid("COMPONENT_UUID");
dto.setSubmitterLogin("rob");

WsCe.Task wsTask = underTest.formatQueue(db.getSession(), dto);

assertThat(wsTask.getType()).isEqualTo("TYPE");
assertThat(wsTask.getId()).isEqualTo("UUID");
assertThat(wsTask.getComponentId()).isEqualTo("COMPONENT_UUID");
assertThat(wsTask.getComponentKey()).isEqualTo("COMPONENT_KEY");
assertThat(wsTask.getComponentName()).isEqualTo("Component Name");
assertThat(wsTask.getStatus()).isEqualTo(WsCe.TaskStatus.IN_PROGRESS);
assertThat(wsTask.getLogs()).isTrue();
assertThat(wsTask.getSubmitterLogin()).isEqualTo("rob");

assertThat(wsTask.hasExecutionTimeMs()).isFalse();
assertThat(wsTask.hasFinishedAt()).isFalse();
}

@Test
public void formatQueue_do_not_fail_if_component_not_found() throws Exception {
CeQueueDto dto = new CeQueueDto();
dto.setUuid("UUID");
dto.setTaskType("TYPE");
dto.setStatus(CeQueueDto.Status.IN_PROGRESS);
dto.setCreatedAt(1_450_000_000_000L);
dto.setComponentUuid("DOES_NOT_EXIST");

WsCe.Task wsTask = underTest.formatQueue(db.getSession(), dto);

assertThat(wsTask.getComponentId()).isEqualTo("DOES_NOT_EXIST");
assertThat(wsTask.hasComponentKey()).isFalse();
assertThat(wsTask.hasComponentName()).isFalse();
}

@Test
public void formatQueues() throws Exception {
CeQueueDto dto1 = new CeQueueDto();
dto1.setUuid("UUID1");
dto1.setTaskType("TYPE1");
dto1.setStatus(CeQueueDto.Status.IN_PROGRESS);
dto1.setCreatedAt(1_450_000_000_000L);

CeQueueDto dto2 = new CeQueueDto();
dto2.setUuid("UUID2");
dto2.setTaskType("TYPE2");
dto2.setStatus(CeQueueDto.Status.PENDING);
dto2.setCreatedAt(1_451_000_000_000L);

List<WsCe.Task> wsTasks = underTest.formatQueue(db.getSession(), asList(dto1, dto2));
assertThat(wsTasks).extracting("id").containsExactly("UUID1", "UUID2");
}

@Test
public void formatActivity() {
CeActivityDto dto = newActivity("UUID", "COMPONENT_UUID", CeActivityDto.Status.FAILED);

WsCe.Task wsTask = underTest.formatActivity(db.getSession(), dto);

assertThat(wsTask.getType()).isEqualTo(CeTaskTypes.REPORT);
assertThat(wsTask.getId()).isEqualTo("UUID");
assertThat(wsTask.getStatus()).isEqualTo(WsCe.TaskStatus.FAILED);
assertThat(wsTask.getLogs()).isFalse();
assertThat(wsTask.getSubmittedAt()).isEqualTo(DateUtils.formatDateTime(new Date(1_450_000_000_000L)));
assertThat(wsTask.getExecutionTimeMs()).isEqualTo(500L);
assertThat(wsTask.getLogs()).isFalse();
}

@Test
public void formatActivity_has_logs() throws IOException {
when(ceLogging.getFile(any(LogFileRef.class))).thenReturn(Optional.of(temp.newFile()));
CeActivityDto dto = newActivity("UUID", "COMPONENT_UUID", CeActivityDto.Status.FAILED);

WsCe.Task wsTask = underTest.formatActivity(db.getSession(), dto);

assertThat(wsTask.getLogs()).isTrue();
}

@Test
public void formatActivities() {
CeActivityDto dto1 = newActivity("UUID1", "COMPONENT_UUID", CeActivityDto.Status.FAILED);
CeActivityDto dto2 = newActivity("UUID2", "COMPONENT_UUID", CeActivityDto.Status.SUCCESS);

List<WsCe.Task> wsTasks = underTest.formatActivity(db.getSession(), asList(dto1, dto2));

assertThat(wsTasks).extracting("id").containsExactly("UUID1", "UUID2");
}

private CeActivityDto newActivity(String taskUuid, String componentUuid, CeActivityDto.Status status) {
CeQueueDto queueDto = new CeQueueDto();
queueDto.setCreatedAt(1_450_000_000_000L);
queueDto.setTaskType(CeTaskTypes.REPORT);
queueDto.setComponentUuid(componentUuid);
queueDto.setUuid(taskUuid);
CeActivityDto activityDto = new CeActivityDto(queueDto);
activityDto.setStatus(status);
activityDto.setExecutionTimeMs(500L);
return activityDto;
}
}

+ 17
- 1
server/sonar-server/src/test/java/org/sonar/server/computation/ws/TaskWsActionTest.java Näytä tiedosto

@@ -19,6 +19,9 @@
*/
package org.sonar.server.computation.ws;

import com.google.common.base.Optional;
import java.io.File;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.utils.System2;
@@ -29,6 +32,8 @@ import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskTypes;
import org.sonar.db.component.ComponentDto;
import org.sonar.server.computation.log.CeLogging;
import org.sonar.server.computation.log.LogFileRef;
import org.sonar.server.exceptions.NotFoundException;
import org.sonar.server.plugins.MimeTypes;
import org.sonar.server.tester.UserSessionRule;
@@ -37,6 +42,9 @@ import org.sonar.server.ws.WsActionTester;
import org.sonarqube.ws.WsCe;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class TaskWsActionTest {

@@ -46,10 +54,16 @@ public class TaskWsActionTest {
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);

TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient());
CeLogging ceLogging = mock(CeLogging.class);
TaskFormatter formatter = new TaskFormatter(dbTester.getDbClient(), ceLogging);
TaskWsAction underTest = new TaskWsAction(dbTester.getDbClient(), formatter, userSession);
WsActionTester tester = new WsActionTester(underTest);

@Before
public void setUp() {
when(ceLogging.getFile(any(LogFileRef.class))).thenReturn(Optional.<File>absent());
}

@Test
public void task_is_in_queue() throws Exception {
userSession.setGlobalPermissions(UserRole.ADMIN);
@@ -80,6 +94,7 @@ public class TaskWsActionTest {
assertThat(taskResponse.getTask().getComponentKey()).isEqualTo(project.key());
assertThat(taskResponse.getTask().getComponentName()).isEqualTo(project.name());
assertThat(taskResponse.getTask().hasExecutionTimeMs()).isFalse();
assertThat(taskResponse.getTask().getLogs()).isFalse();
}

@Test
@@ -112,6 +127,7 @@ public class TaskWsActionTest {
assertThat(taskResponse.getTask().getComponentKey()).isEqualTo(project.key());
assertThat(taskResponse.getTask().getComponentName()).isEqualTo(project.name());
assertThat(taskResponse.getTask().getExecutionTimeMs()).isEqualTo(500L);
assertThat(taskResponse.getTask().getLogs()).isFalse();
}

@Test(expected = NotFoundException.class)

+ 2
- 3
sonar-application/src/main/java/org/sonar/application/AppLogging.java Näytä tiedosto

@@ -27,7 +27,6 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.ConsoleAppender;
import ch.qos.logback.core.FileAppender;

import org.slf4j.LoggerFactory;
import org.sonar.process.LogbackHelper;
import org.sonar.process.Props;
@@ -71,7 +70,7 @@ class AppLogging {
}

private void configureConsole(LoggerContext loggerContext) {
ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(loggerContext, CONSOLE_APPENDER, "%msg%n");
ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(loggerContext, CONSOLE_APPENDER, "%msg%n", null);
Logger consoleLogger = loggerContext.getLogger(CONSOLE_LOGGER);
consoleLogger.setAdditive(false);
consoleLogger.addAppender(consoleAppender);
@@ -96,7 +95,7 @@ class AppLogging {
}

private void configureRoot(LoggerContext loggerContext) {
ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(loggerContext, "ROOT_CONSOLE", APP_PATTERN);
ConsoleAppender<ILoggingEvent> consoleAppender = helper.newConsoleAppender(loggerContext, "ROOT_CONSOLE", APP_PATTERN, null);
Logger rootLogger = loggerContext.getLogger(Logger.ROOT_LOGGER_NAME);
rootLogger.setLevel(Level.INFO);
rootLogger.addAppender(consoleAppender);

+ 0
- 3
sonar-core/src/main/java/org/sonar/core/config/PurgeConstants.java Näytä tiedosto

@@ -22,10 +22,7 @@ package org.sonar.core.config;

public interface PurgeConstants {

String PLUGIN_KEY = "dbcleaner";
String PLUGIN_NAME = "DbCleaner";
String PROPERTY_CLEAN_DIRECTORY = "sonar.dbcleaner.cleanDirectory";

String HOURS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_DAY = "sonar.dbcleaner.hoursBeforeKeepingOnlyOneSnapshotByDay";
String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_WEEK = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByWeek";
String WEEKS_BEFORE_KEEPING_ONLY_ONE_SNAPSHOT_BY_MONTH = "sonar.dbcleaner.weeksBeforeKeepingOnlyOneSnapshotByMonth";

+ 92
- 5
sonar-ws/src/main/gen-java/org/sonarqube/ws/WsCe.java Näytä tiedosto

@@ -3851,6 +3851,16 @@ public final class WsCe {
* <code>optional int64 executionTimeMs = 12;</code>
*/
long getExecutionTimeMs();

/**
* <code>optional bool logs = 13;</code>
*/
boolean hasLogs();

/**
* <code>optional bool logs = 13;</code>
*/
boolean getLogs();
}
/**
* Protobuf type {@code sonarqube.ws.ce.Task}
@@ -3876,6 +3886,7 @@ public final class WsCe {
finishedAt_ = "";
isLastFinished_ = false;
executionTimeMs_ = 0L;
logs_ = false;
}

@java.lang.Override
@@ -3980,6 +3991,11 @@ public final class WsCe {
executionTimeMs_ = input.readInt64();
break;
}
case 104: {
bitField0_ |= 0x00001000;
logs_ = input.readBool();
break;
}
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -4430,6 +4446,23 @@ public final class WsCe {
return executionTimeMs_;
}

public static final int LOGS_FIELD_NUMBER = 13;
private boolean logs_;

/**
* <code>optional bool logs = 13;</code>
*/
public boolean hasLogs() {
return ((bitField0_ & 0x00001000) == 0x00001000);
}

/**
* <code>optional bool logs = 13;</code>
*/
public boolean getLogs() {
return logs_;
}

private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
@@ -4478,6 +4511,9 @@ public final class WsCe {
if (((bitField0_ & 0x00000800) == 0x00000800)) {
output.writeInt64(12, executionTimeMs_);
}
if (((bitField0_ & 0x00001000) == 0x00001000)) {
output.writeBool(13, logs_);
}
unknownFields.writeTo(output);
}

@@ -4535,6 +4571,10 @@ public final class WsCe {
size += com.google.protobuf.CodedOutputStream
.computeInt64Size(12, executionTimeMs_);
}
if (((bitField0_ & 0x00001000) == 0x00001000)) {
size += com.google.protobuf.CodedOutputStream
.computeBoolSize(13, logs_);
}
size += unknownFields.getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -4671,6 +4711,8 @@ public final class WsCe {
bitField0_ = (bitField0_ & ~0x00000400);
executionTimeMs_ = 0L;
bitField0_ = (bitField0_ & ~0x00000800);
logs_ = false;
bitField0_ = (bitField0_ & ~0x00001000);
return this;
}

@@ -4743,6 +4785,10 @@ public final class WsCe {
to_bitField0_ |= 0x00000800;
}
result.executionTimeMs_ = executionTimeMs_;
if (((from_bitField0_ & 0x00001000) == 0x00001000)) {
to_bitField0_ |= 0x00001000;
}
result.logs_ = logs_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -4813,6 +4859,9 @@ public final class WsCe {
if (other.hasExecutionTimeMs()) {
setExecutionTimeMs(other.getExecutionTimeMs());
}
if (other.hasLogs()) {
setLogs(other.getLogs());
}
this.mergeUnknownFields(other.unknownFields);
onChanged();
return this;
@@ -5625,6 +5674,42 @@ public final class WsCe {
return this;
}

private boolean logs_;

/**
* <code>optional bool logs = 13;</code>
*/
public boolean hasLogs() {
return ((bitField0_ & 0x00001000) == 0x00001000);
}

/**
* <code>optional bool logs = 13;</code>
*/
public boolean getLogs() {
return logs_;
}

/**
* <code>optional bool logs = 13;</code>
*/
public Builder setLogs(boolean value) {
bitField0_ |= 0x00001000;
logs_ = value;
onChanged();
return this;
}

/**
* <code>optional bool logs = 13;</code>
*/
public Builder clearLogs() {
bitField0_ = (bitField0_ & ~0x00001000);
logs_ = false;
onChanged();
return this;
}

// @@protoc_insertion_point(builder_scope:sonarqube.ws.ce.Task)
}

@@ -5717,16 +5802,17 @@ public final class WsCe {
"asks\030\002 \003(\0132\025.sonarqube.ws.ce.Task\"_\n\017Pro" +
"jectResponse\022$\n\005queue\030\001 \003(\0132\025.sonarqube." +
"ws.ce.Task\022&\n\007current\030\002 \001(\0132\025.sonarqube.",
"ws.ce.Task\"\224\002\n\004Task\022\n\n\002id\030\001 \001(\t\022\014\n\004type\030" +
"ws.ce.Task\"\242\002\n\004Task\022\n\n\002id\030\001 \001(\t\022\014\n\004type\030" +
"\002 \001(\t\022\023\n\013componentId\030\003 \001(\t\022\024\n\014componentK" +
"ey\030\004 \001(\t\022\025\n\rcomponentName\030\005 \001(\t\022+\n\006statu" +
"s\030\006 \001(\0162\033.sonarqube.ws.ce.TaskStatus\022\023\n\013" +
"submittedAt\030\007 \001(\t\022\026\n\016submitterLogin\030\010 \001(" +
"\t\022\021\n\tstartedAt\030\t \001(\t\022\022\n\nfinishedAt\030\n \001(\t" +
"\022\026\n\016isLastFinished\030\013 \001(\010\022\027\n\017executionTim" +
"eMs\030\014 \001(\003*Q\n\nTaskStatus\022\013\n\007PENDING\020\000\022\017\n\013" +
"IN_PROGRESS\020\001\022\013\n\007SUCCESS\020\002\022\n\n\006FAILED\020\003\022\014" +
"\n\010CANCELED\020\004B\032\n\020org.sonarqube.wsB\004WsCeH\001"
"eMs\030\014 \001(\003\022\014\n\004logs\030\r \001(\010*Q\n\nTaskStatus\022\013\n" +
"\007PENDING\020\000\022\017\n\013IN_PROGRESS\020\001\022\013\n\007SUCCESS\020\002" +
"\022\n\n\006FAILED\020\003\022\014\n\010CANCELED\020\004B\032\n\020org.sonarq",
"ube.wsB\004WsCeH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() {
@@ -5776,7 +5862,8 @@ public final class WsCe {
internal_static_sonarqube_ws_ce_Task_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_sonarqube_ws_ce_Task_descriptor,
new java.lang.String[] { "Id", "Type", "ComponentId", "ComponentKey", "ComponentName", "Status", "SubmittedAt", "SubmitterLogin", "StartedAt", "FinishedAt", "IsLastFinished", "ExecutionTimeMs", });
new java.lang.String[] {"Id", "Type", "ComponentId", "ComponentKey", "ComponentName", "Status", "SubmittedAt", "SubmitterLogin", "StartedAt", "FinishedAt", "IsLastFinished",
"ExecutionTimeMs", "Logs",});
org.sonarqube.ws.Common.getDescriptor();
}


+ 1
- 0
sonar-ws/src/main/protobuf/ws-ce.proto Näytä tiedosto

@@ -67,6 +67,7 @@ message Task {
optional string finishedAt = 10;
optional bool isLastFinished = 11;
optional int64 executionTimeMs = 12;
optional bool logs = 13;
}

enum TaskStatus {

Loading…
Peruuta
Tallenna