SONAR-6057 Make log rotation more configurable for production

6 files changed, 401 insertions, 68 deletions

diff --git a/server/sonar-process/pom.xml b/server/sonar-process/pom.xml
index f4da18c6757..03bbf3d7c81 100644
--- a/server/sonar-process/pom.xml
+++ b/server/sonar-process/pom.xml
@@ -12,16 +12,10 @@
   <name>SonarQube :: Process</name>
 
   <dependencies>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-    </dependency>
     <dependency>
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-classic</artifactId>
     </dependency>
-
     <dependency>
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
@@ -34,6 +28,7 @@
       <groupId>commons-lang</groupId>
       <artifactId>commons-lang</artifactId>
     </dependency>
+
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
diff --git a/server/sonar-process/src/main/java/org/sonar/process/LogbackHelper.java b/server/sonar-process/src/main/java/org/sonar/process/LogbackHelper.java
new file mode 100644
index 00000000000..862fc22c7a4
--- /dev/null
+++ b/server/sonar-process/src/main/java/org/sonar/process/LogbackHelper.java
@@ -0,0 +1,215 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.classic.jul.LevelChangePropagator;
+import ch.qos.logback.classic.spi.LoggerContextListener;
+import ch.qos.logback.core.ConsoleAppender;
+import ch.qos.logback.core.Context;
+import ch.qos.logback.core.FileAppender;
+import ch.qos.logback.core.joran.spi.JoranException;
+import ch.qos.logback.core.rolling.FixedWindowRollingPolicy;
+import ch.qos.logback.core.rolling.RollingFileAppender;
+import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy;
+import ch.qos.logback.core.rolling.TimeBasedRollingPolicy;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+/**
+ * Helps to configure Logback in a programmatic way, without using XML.
+ */
+public class LogbackHelper {
+
+  public static final String ROLLING_POLICY_PROPERTY = "sonar.log.rollingPolicy";
+  public static final String MAX_FILES_PROPERTY = "sonar.log.maxFiles";
+
+  public LoggerContext getRootContext() {
+    Logger rootLogger = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
+    return rootLogger.getLoggerContext();
+  }
+
+  public LoggerContextListener enableJulChangePropagation(LoggerContext loggerContext) {
+    LevelChangePropagator propagator = new LevelChangePropagator();
+    propagator.setContext(loggerContext);
+    propagator.start();
+    loggerContext.addListener(propagator);
+    return propagator;
+  }
+
+  public ConsoleAppender newConsoleAppender(Context loggerContext, String name, String pattern) {
+    PatternLayoutEncoder consoleEncoder = new PatternLayoutEncoder();
+    consoleEncoder.setContext(loggerContext);
+    consoleEncoder.setPattern(pattern);
+    consoleEncoder.start();
+    ConsoleAppender consoleAppender = new ConsoleAppender();
+    consoleAppender.setContext(loggerContext);
+    consoleAppender.setEncoder(consoleEncoder);
+    consoleAppender.setName(name);
+    consoleAppender.setTarget("System.out");
+    consoleAppender.start();
+    return consoleAppender;
+  }
+
+  public Logger configureLogger(LoggerContext loggerContext, String loggerName, Level level) {
+    Logger logger = loggerContext.getLogger(loggerName);
+    logger.setLevel(level);
+    return logger;
+  }
+
+  /**
+   * Generally used to reset logback in logging tests
+   */
+  public void resetFromXml(String xmlResourcePath) throws JoranException {
+    LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+    JoranConfigurator configurator = new JoranConfigurator();
+    configurator.setContext(context);
+    context.reset();
+    configurator.doConfigure(LogbackHelper.class.getResource(xmlResourcePath));
+  }
+
+  public RollingPolicy createRollingPolicy(Context ctx, Props props, String filenamePrefix) {
+    String rollingPolicy = props.value(ROLLING_POLICY_PROPERTY, "time:yyyy-MM-dd");
+    int maxFiles = props.valueAsInt(MAX_FILES_PROPERTY, 7);
+    File logsDir = props.nonNullValueAsFile(ProcessConstants.PATH_LOGS);
+
+    if (rollingPolicy.startsWith("time:")) {
+      return new TimeRollingPolicy(ctx, filenamePrefix, logsDir, maxFiles, StringUtils.substringAfter(rollingPolicy, "time:"));
+
+    } else if (rollingPolicy.startsWith("size:")) {
+      return new SizeRollingPolicy(ctx, filenamePrefix, logsDir, maxFiles, StringUtils.substringAfter(rollingPolicy, "size:"));
+
+    } else if (rollingPolicy.equals("none")) {
+      return new NoRollingPolicy(ctx, filenamePrefix, logsDir, maxFiles);
+
+    } else {
+      throw new MessageException(String.format("Unsupported value for property %s: %s", ROLLING_POLICY_PROPERTY, rollingPolicy));
+    }
+  }
+
+  public static abstract class RollingPolicy {
+    protected final Context context;
+    protected final String filenamePrefix;
+    protected final File logsDir;
+    protected final int maxFiles;
+
+    public RollingPolicy(Context context, String filenamePrefix, File logsDir, int maxFiles) {
+      this.context = context;
+      this.filenamePrefix = filenamePrefix;
+      this.logsDir = logsDir;
+      this.maxFiles = maxFiles;
+    }
+
+    public abstract FileAppender createAppender(String appenderName);
+  }
+
+  /**
+   * Log files are not rotated, for example for unix command logrotate is in place.
+   */
+  static class NoRollingPolicy extends RollingPolicy {
+    NoRollingPolicy(Context context, String filenamePrefix, File logsDir, int maxFiles) {
+      super(context, filenamePrefix, logsDir, maxFiles);
+    }
+
+    @Override
+    public FileAppender createAppender(String appenderName) {
+      FileAppender appender = new FileAppender<>();
+      appender.setContext(context);
+      appender.setFile(new File(logsDir, filenamePrefix + ".log").getAbsolutePath());
+      appender.setName(appenderName);
+      return appender;
+    }
+  }
+
+  /**
+   * Log files are rotated according to time (one file per day, month or year).
+   * See http://logback.qos.ch/manual/appenders.html#TimeBasedRollingPolicy
+   */
+  static class TimeRollingPolicy extends RollingPolicy {
+    private final String datePattern;
+
+    public TimeRollingPolicy(Context context, String filenamePrefix, File logsDir, int maxFiles, String datePattern) {
+      super(context, filenamePrefix, logsDir, maxFiles);
+      this.datePattern = datePattern;
+    }
+
+    @Override
+    public FileAppender createAppender(String appenderName) {
+      RollingFileAppender appender = new RollingFileAppender();
+      appender.setContext(context);
+      appender.setName(appenderName);
+      appender.setFile(new File(logsDir, filenamePrefix + ".log").getAbsolutePath());
+
+      TimeBasedRollingPolicy rollingPolicy = new TimeBasedRollingPolicy();
+      rollingPolicy.setContext(context);
+      rollingPolicy.setFileNamePattern(filenamePrefix + ".%d{" + datePattern + "}.log");
+      rollingPolicy.setMaxHistory(maxFiles);
+      rollingPolicy.setParent(appender);
+      rollingPolicy.start();
+      appender.setRollingPolicy(rollingPolicy);
+
+      return appender;
+    }
+  }
+
+  /**
+   * Log files are rotated according to their size.
+   * See http://logback.qos.ch/manual/appenders.html#FixedWindowRollingPolicy
+   */
+  static class SizeRollingPolicy extends RollingPolicy {
+    private final String size;
+
+    SizeRollingPolicy(Context context, String filenamePrefix, File logsDir, int maxFiles, String parameter) {
+      super(context, filenamePrefix, logsDir, maxFiles);
+      this.size = parameter;
+    }
+
+    @Override
+    public FileAppender createAppender(String appenderName) {
+      RollingFileAppender appender = new RollingFileAppender();
+      appender.setContext(context);
+      appender.setName(appenderName);
+      appender.setFile(new File(logsDir, filenamePrefix + ".log").getAbsolutePath());
+
+      SizeBasedTriggeringPolicy trigger = new SizeBasedTriggeringPolicy(size);
+      trigger.setContext(context);
+      trigger.start();
+      appender.setTriggeringPolicy(trigger);
+
+      FixedWindowRollingPolicy rollingPolicy = new FixedWindowRollingPolicy();
+      rollingPolicy.setContext(context);
+      rollingPolicy.setFileNamePattern(filenamePrefix + ".%i.log");
+      rollingPolicy.setMaxIndex(1);
+      rollingPolicy.setMaxIndex(maxFiles);
+      rollingPolicy.setMaxIndex(maxFiles);
+      rollingPolicy.setParent(appender);
+      rollingPolicy.start();
+      appender.setRollingPolicy(rollingPolicy);
+
+      return appender;
+    }
+  }
+}
diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessLogging.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessLogging.java
deleted file mode 100644
index fe7f647d2e9..00000000000
--- a/server/sonar-process/src/main/java/org/sonar/process/ProcessLogging.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.process;
-
-import ch.qos.logback.classic.Logger;
-import ch.qos.logback.classic.LoggerContext;
-import ch.qos.logback.classic.joran.JoranConfigurator;
-import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.core.Appender;
-import ch.qos.logback.core.joran.spi.JoranException;
-import ch.qos.logback.core.util.StatusPrinter;
-import org.slf4j.LoggerFactory;
-
-public class ProcessLogging {
-
-  public void configure(Props props, String logbackXmlResource) {
-    LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
-    try {
-      JoranConfigurator configurator = new JoranConfigurator();
-      configurator.setContext(context);
-      context.reset();
-      context.putProperty(ProcessConstants.PATH_LOGS, props.nonNullValue(ProcessConstants.PATH_LOGS));
-      doConfigure(configurator, logbackXmlResource);
-    } catch (JoranException ignored) {
-      // StatusPrinter will handle this
-    }
-    StatusPrinter.printInCaseOfErrorsOrWarnings(context);
-  }
-
-  public void addConsoleAppender() {
-    Logger consoleLogger = (Logger) LoggerFactory.getLogger("console");
-    Appender<ILoggingEvent> consoleAppender = consoleLogger.getAppender("CONSOLE");
-
-    Logger gobblerLogger = (Logger) LoggerFactory.getLogger("gobbler");
-    gobblerLogger.addAppender(consoleAppender);
-  }
-
-  /**
-   * Extracted only for unit testing
-   */
-  void doConfigure(JoranConfigurator configurator, String logbackXmlResource) throws JoranException {
-    configurator.doConfigure(getClass().getResource(logbackXmlResource));
-  }
-}
diff --git a/server/sonar-process/src/test/java/org/sonar/process/LogbackHelperTest.java b/server/sonar-process/src/test/java/org/sonar/process/LogbackHelperTest.java
new file mode 100644
index 00000000000..d761a2dca76
--- /dev/null
+++ b/server/sonar-process/src/test/java/org/sonar/process/LogbackHelperTest.java
@@ -0,0 +1,172 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
+import ch.qos.logback.classic.spi.LoggerContextListener;
+import ch.qos.logback.core.Appender;
+import ch.qos.logback.core.ConsoleAppender;
+import ch.qos.logback.core.FileAppender;
+import ch.qos.logback.core.rolling.FixedWindowRollingPolicy;
+import ch.qos.logback.core.rolling.RollingFileAppender;
+import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy;
+import ch.qos.logback.core.rolling.TimeBasedRollingPolicy;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.util.Properties;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.fail;
+
+public class LogbackHelperTest {
+
+  Props props = new Props(new Properties());
+  LogbackHelper sut = new LogbackHelper();
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  @Before
+  public void setUp() throws Exception {
+    File dir = temp.newFolder();
+    props.set(ProcessConstants.PATH_LOGS, dir.getAbsolutePath());
+  }
+
+  @AfterClass
+  public static void resetLogback() throws Exception {
+    new LogbackHelper().resetFromXml("/logback-test.xml");
+  }
+
+  @Test
+  public void getRootContext() {
+    assertThat(sut.getRootContext()).isNotNull();
+  }
+
+  @Test
+  public void enableJulChangePropagation() {
+    LoggerContext ctx = sut.getRootContext();
+    int countListeners = ctx.getCopyOfListenerList().size();
+
+    LoggerContextListener propagator = sut.enableJulChangePropagation(ctx);
+    assertThat(ctx.getCopyOfListenerList().size()).isEqualTo(countListeners + 1);
+
+    ctx.removeListener(propagator);
+  }
+
+  @Test
+  public void newConsoleAppender() throws Exception {
+    LoggerContext ctx = sut.getRootContext();
+    ConsoleAppender appender = sut.newConsoleAppender(ctx, "MY_APPENDER", "%msg%n");
+
+    assertThat(appender.getName()).isEqualTo("MY_APPENDER");
+    assertThat(appender.getContext()).isSameAs(ctx);
+    assertThat(appender.isStarted()).isTrue();
+    assertThat(((PatternLayoutEncoder) appender.getEncoder()).getPattern()).isEqualTo("%msg%n");
+  }
+
+  @Test
+  public void configureLogger() throws Exception {
+    LoggerContext ctx = sut.getRootContext();
+
+    Logger logger = sut.configureLogger(ctx, "my_logger", Level.WARN);
+
+    assertThat(logger.getLevel()).isEqualTo(Level.WARN);
+    assertThat(logger.getName()).isEqualTo("my_logger");
+  }
+
+  @Test
+  public void createRollingPolicy_defaults() throws Exception {
+    LoggerContext ctx = sut.getRootContext();
+    LogbackHelper.RollingPolicy policy = sut.createRollingPolicy(ctx, props, "sonar");
+    FileAppender appender = policy.createAppender("SONAR_FILE");
+    assertThat(appender).isInstanceOf(RollingFileAppender.class);
+
+    // max 5 daily files
+    RollingFileAppender fileAppender = (RollingFileAppender) appender;
+    TimeBasedRollingPolicy triggeringPolicy = (TimeBasedRollingPolicy) fileAppender.getTriggeringPolicy();
+    assertThat(triggeringPolicy.getMaxHistory()).isEqualTo(7);
+    assertThat(triggeringPolicy.getFileNamePattern()).isEqualTo("sonar.%d{yyyy-MM-dd}.log");
+  }
+
+  @Test
+  public void createRollingPolicy_none() throws Exception {
+    props.set("sonar.log.rollingPolicy", "none");
+    LoggerContext ctx = sut.getRootContext();
+    LogbackHelper.RollingPolicy policy = sut.createRollingPolicy(ctx, props, "sonar");
+
+    Appender appender = policy.createAppender("SONAR_FILE");
+    assertThat(appender).isNotInstanceOf(RollingFileAppender.class).isInstanceOf(FileAppender.class);
+  }
+
+  @Test
+  public void createRollingPolicy_size() throws Exception {
+    props.set("sonar.log.rollingPolicy", "size:1MB");
+    props.set("sonar.log.maxFiles", "20");
+    LoggerContext ctx = sut.getRootContext();
+    LogbackHelper.RollingPolicy policy = sut.createRollingPolicy(ctx, props, "sonar");
+
+    Appender appender = policy.createAppender("SONAR_FILE");
+    assertThat(appender).isInstanceOf(RollingFileAppender.class);
+
+    // max 20 files of 1Mb
+    RollingFileAppender fileAppender = (RollingFileAppender) appender;
+    FixedWindowRollingPolicy rollingPolicy = (FixedWindowRollingPolicy) fileAppender.getRollingPolicy();
+    assertThat(rollingPolicy.getMaxIndex()).isEqualTo(20);
+    assertThat(rollingPolicy.getFileNamePattern()).isEqualTo("sonar.%i.log");
+    SizeBasedTriggeringPolicy triggeringPolicy = (SizeBasedTriggeringPolicy) fileAppender.getTriggeringPolicy();
+    assertThat(triggeringPolicy.getMaxFileSize()).isEqualTo("1MB");
+  }
+
+  @Test
+  public void createRollingPolicy_time() throws Exception {
+    props.set("sonar.log.rollingPolicy", "time:yyyy-MM");
+    props.set("sonar.log.maxFiles", "20");
+
+    LoggerContext ctx = sut.getRootContext();
+    LogbackHelper.RollingPolicy policy = sut.createRollingPolicy(ctx, props, "sonar");
+
+    RollingFileAppender appender = (RollingFileAppender) policy.createAppender("SONAR_FILE");
+
+    // max 5 monthly files
+    TimeBasedRollingPolicy triggeringPolicy = (TimeBasedRollingPolicy) appender.getTriggeringPolicy();
+    assertThat(triggeringPolicy.getMaxHistory()).isEqualTo(20);
+    assertThat(triggeringPolicy.getFileNamePattern()).isEqualTo("sonar.%d{yyyy-MM}.log");
+  }
+
+  @Test
+  public void createRollingPolicy_fail_if_unknown_policy() throws Exception {
+    props.set("sonar.log.rollingPolicy", "unknown:foo");
+    try {
+      LoggerContext ctx = sut.getRootContext();
+      sut.createRollingPolicy(ctx, props, "sonar");
+      fail();
+    } catch (MessageException e) {
+      assertThat(e).hasMessage("Unsupported value for property sonar.log.rollingPolicy: unknown:foo");
+    }
+  }
+}
diff --git a/server/sonar-process/src/test/resources/logback-test.xml b/server/sonar-process/src/test/resources/logback-test.xml
new file mode 100644
index 00000000000..f2d5a6a466f
--- /dev/null
+++ b/server/sonar-process/src/test/resources/logback-test.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<configuration debug="false">
+
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+    </encoder>
+  </appender>
+
+  <root level="debug">
+    <appender-ref ref="STDOUT"/>
+  </root>
+</configuration>
diff --git a/server/sonar-process/src/test/resources/org/sonar/process/LoggingTest/logback-access.xml b/server/sonar-process/src/test/resources/org/sonar/process/LoggingTest/logback-access.xml
deleted file mode 100644
index 298193e01fa..00000000000
--- a/server/sonar-process/src/test/resources/org/sonar/process/LoggingTest/logback-access.xml
+++ /dev/null
@@ -1 +0,0 @@
-<configuration/>