From: Sébastien Lesaint Date: Wed, 13 Jan 2016 10:24:01 +0000 (+0100) Subject: SONAR-7168 fileSystem must be reset during a restart X-Git-Tag: 5.4-M8~12 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=e7c4a306ac47843029d183d8e3f5ca7730184cf2;p=sonarqube.git SONAR-7168 fileSystem must be reset during a restart deletion of temp directory is required for uninstalling plugins but also to cleanly start web server (eg. deleting unsuccessfully processed analysis reports) --- diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/FileSystem.java b/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/FileSystem.java new file mode 100644 index 00000000000..4efe53bf489 --- /dev/null +++ b/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/FileSystem.java @@ -0,0 +1,29 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.process.monitor; + +import java.io.File; +import java.io.IOException; + +public interface FileSystem { + void reset() throws IOException; + + File getTempDir(); +} diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/JavaProcessLauncher.java b/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/JavaProcessLauncher.java index 2f9490b1844..019d6388cd9 100644 --- a/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/JavaProcessLauncher.java +++ b/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/JavaProcessLauncher.java @@ -39,16 +39,20 @@ public class JavaProcessLauncher { private final File tempDir; private final AllProcessesCommands allProcessesCommands; - public JavaProcessLauncher(Timeouts timeouts, File tempDir, AllProcessesCommands allProcessesCommands) { + public JavaProcessLauncher(Timeouts timeouts, File tempDir) { this.timeouts = timeouts; this.tempDir = tempDir; - this.allProcessesCommands = allProcessesCommands; + this.allProcessesCommands = new AllProcessesCommands(tempDir); } public void close() { allProcessesCommands.close(); } + public ProcessCommands getProcessCommand(int processNumber, boolean clean) { + return allProcessesCommands.getProcessCommand(processNumber, clean); + } + ProcessRef launch(JavaCommand command) { Process process = null; try { diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/Monitor.java b/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/Monitor.java index 8c3e459106f..481495b7d14 100644 --- a/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/Monitor.java +++ b/server/sonar-process-monitor/src/main/java/org/sonar/process/monitor/Monitor.java @@ -20,6 +20,7 @@ package org.sonar.process.monitor; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -28,7 +29,6 @@ import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.process.AllProcessesCommands; import org.sonar.process.Lifecycle; import org.sonar.process.Lifecycle.State; import org.sonar.process.ProcessCommands; @@ -43,32 +43,34 @@ public class Monitor { private static int restartorInstanceCounter = 0; - private final AllProcessesCommands allProcessesCommands; - private final JavaProcessLauncher launcher; - + private final FileSystem fileSystem; private final SystemExit systemExit; + private final boolean watchForHardStop; private final Thread shutdownHook = new Thread(new MonitorShutdownHook(), "Monitor Shutdown Hook"); private final List watcherThreads = new CopyOnWriteArrayList<>(); private final Lifecycle lifecycle = new Lifecycle(); + private final TerminatorThread terminator = new TerminatorThread(); private final RestartRequestWatcherThread restartWatcher = new RestartRequestWatcherThread(); @CheckForNull private List javaCommands; @CheckForNull + private JavaProcessLauncher launcher; + @CheckForNull private RestartorThread restartor; @CheckForNull HardStopWatcherThread hardStopWatcher; static int nextProcessId = 1; - Monitor(File tempDir, SystemExit exit) { - this.allProcessesCommands = new AllProcessesCommands(tempDir); - this.launcher = new JavaProcessLauncher(TIMEOUTS, tempDir, allProcessesCommands); + Monitor(FileSystem fileSystem, SystemExit exit, boolean watchForHardStop) { + this.fileSystem = fileSystem; this.systemExit = exit; + this.watchForHardStop = watchForHardStop; } - public static Monitor create(File tempDir) { - return new Monitor(tempDir, new SystemExit()); + public static Monitor create(FileSystem fileSystem, boolean watchForHardStop) { + return new Monitor(fileSystem, new SystemExit(), watchForHardStop); } /** @@ -88,6 +90,8 @@ public class Monitor { // intercepts CTRL-C Runtime.getRuntime().addShutdownHook(shutdownHook); + + // start watching for restart requested by child process this.restartWatcher.start(); this.javaCommands = commands; @@ -97,12 +101,42 @@ public class Monitor { private void startProcesses() { // do no start any child process if not in state INIT or RESTARTING (a stop could be in progress too) if (lifecycle.tryToMoveTo(State.STARTING)) { + resetFileSystem(); + startAndMonitorProcesses(); stopIfAnyProcessDidNotStart(); + watchForHardStop(); + } + } + + private void watchForHardStop() { + if (!watchForHardStop) { + return; + } + + if (this.hardStopWatcher != null) { + this.hardStopWatcher.stopWatching(); + awaitTermination(this.hardStopWatcher); + } + ProcessCommands processCommand = this.launcher.getProcessCommand(CURRENT_PROCESS_NUMBER, true); + this.hardStopWatcher = new HardStopWatcherThread(processCommand); + this.hardStopWatcher.start(); + } + + private void resetFileSystem() { + // since JavaLauncher depends on temp directory, which is reset below, we need to close it first + closeJavaLauncher(); + try { + fileSystem.reset(); + } catch (IOException e) { + // failed to reset FileSystem + throw new RuntimeException("Failed to reset file system", e); } } private void startAndMonitorProcesses() { + File tempDir = fileSystem.getTempDir(); + this.launcher = new JavaProcessLauncher(TIMEOUTS, tempDir); for (JavaCommand command : javaCommands) { try { ProcessRef processRef = launcher.launch(command); @@ -115,10 +149,11 @@ public class Monitor { } } - public void watchForHardStop() { - ProcessCommands processCommand = this.allProcessesCommands.getProcessCommand(CURRENT_PROCESS_NUMBER, true); - this.hardStopWatcher = new HardStopWatcherThread(processCommand); - this.hardStopWatcher.start(); + private void closeJavaLauncher() { + if (this.launcher != null) { + this.launcher.close(); + this.launcher = null; + } } private void monitor(ProcessRef processRef) { @@ -213,7 +248,7 @@ public class Monitor { Runtime.getRuntime().removeShutdownHook(shutdownHook); } // cleanly close JavaLauncher - launcher.close(); + closeJavaLauncher(); } } @@ -227,6 +262,7 @@ public class Monitor { } private void stopAsync(State stoppingState) { + assert stoppingState == State.STOPPING || stoppingState == State.HARD_STOPPING; if (lifecycle.tryToMoveTo(stoppingState)) { terminator.start(); } @@ -310,6 +346,7 @@ public class Monitor { public class HardStopWatcherThread extends Thread { private final ProcessCommands processCommands; + private boolean watch = true; public HardStopWatcherThread(ProcessCommands processCommands) { super("Hard stop watcher"); @@ -318,7 +355,7 @@ public class Monitor { @Override public void run() { - while (lifecycle.getState() != Lifecycle.State.STOPPED) { + while (watch && lifecycle.getState() != Lifecycle.State.STOPPED) { if (processCommands.askedForStop()) { trace("Stopping process"); Monitor.this.stop(); @@ -332,6 +369,9 @@ public class Monitor { } } + public void stopWatching() { + this.watch = false; + } } private void stopProcesses() { diff --git a/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/JavaProcessLauncherTest.java b/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/JavaProcessLauncherTest.java index 20053415c15..af32f3f759b 100644 --- a/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/JavaProcessLauncherTest.java +++ b/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/JavaProcessLauncherTest.java @@ -23,7 +23,6 @@ import java.io.File; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.sonar.process.AllProcessesCommands; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -37,7 +36,7 @@ public class JavaProcessLauncherTest { public void fail_to_launch() throws Exception { File tempDir = temp.newFolder(); JavaCommand command = new JavaCommand("test"); - JavaProcessLauncher launcher = new JavaProcessLauncher(new Timeouts(), tempDir, new AllProcessesCommands(tempDir)); + JavaProcessLauncher launcher = new JavaProcessLauncher(new Timeouts(), tempDir); try { // command is not correct (missing options), java.lang.ProcessBuilder#start() // throws an exception diff --git a/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/MonitorTest.java b/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/MonitorTest.java index 713ace625dc..ea25a399cfe 100644 --- a/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/MonitorTest.java +++ b/server/sonar-process-monitor/src/test/java/org/sonar/process/monitor/MonitorTest.java @@ -45,16 +45,23 @@ import org.sonar.process.NetworkUtils; import org.sonar.process.ProcessCommands; import org.sonar.process.SystemExit; +import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.sonar.process.monitor.MonitorTest.HttpProcessClientAssert.assertThat; public class MonitorTest { - static File testJar; - Monitor monitor; - SystemExit exit = mock(SystemExit.class); + private static File testJar; + + private FileSystem fileSystem = mock(FileSystem.class); + private SystemExit exit = mock(SystemExit.class); + + private Monitor underTest; /** * Find the JAR file containing the test apps. Classes can't be moved in sonar-process-monitor because @@ -105,8 +112,8 @@ public class MonitorTest { @After public void tearDown() { try { - if (monitor != null) { - monitor.stop(); + if (underTest != null) { + underTest.stop(); } } catch (Throwable ignored) { } @@ -114,9 +121,9 @@ public class MonitorTest { @Test public void fail_to_start_if_no_commands() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); try { - monitor.start(Collections.emptyList()); + underTest.start(Collections.emptyList()); fail(); } catch (IllegalArgumentException e) { assertThat(e).hasMessage("At least one command is required"); @@ -125,51 +132,52 @@ public class MonitorTest { @Test public void fail_to_start_multiple_times() throws Exception { - monitor = newDefaultMonitor(tempDir); - monitor.start(Arrays.asList(newStandardProcessCommand())); + underTest = newDefaultMonitor(tempDir); + underTest.start(singletonList(newStandardProcessCommand())); boolean failed = false; try { - monitor.start(Arrays.asList(newStandardProcessCommand())); + underTest.start(singletonList(newStandardProcessCommand())); } catch (IllegalStateException e) { failed = e.getMessage().equals("Can not start multiple times"); } - monitor.stop(); + underTest.stop(); assertThat(failed).isTrue(); } @Test public void start_then_stop_gracefully() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); HttpProcessClient client = new HttpProcessClient(tempDir, "test"); // blocks until started - monitor.start(Arrays.asList(client.newCommand())); + underTest.start(singletonList(client.newCommand())); assertThat(client).isReady() .wasStartedBefore(System.currentTimeMillis()); // blocks until stopped - monitor.stop(); + underTest.stop(); assertThat(client) .isNotReady() .wasGracefullyTerminated(); - assertThat(monitor.getState()).isEqualTo(State.STOPPED); + assertThat(underTest.getState()).isEqualTo(State.STOPPED); + verify(fileSystem).reset(); } @Test public void start_then_stop_sequence_of_commands() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); HttpProcessClient p1 = new HttpProcessClient(tempDir, "p1"); HttpProcessClient p2 = new HttpProcessClient(tempDir, "p2"); - monitor.start(Arrays.asList(p1.newCommand(), p2.newCommand())); + underTest.start(Arrays.asList(p1.newCommand(), p2.newCommand())); // start p2 when p1 is fully started (ready) assertThat(p1) .isReady() .wasStartedBefore(p2); assertThat(p2) - .isReady(); + .isReady(); - monitor.stop(); + underTest.stop(); // stop in inverse order assertThat(p1) @@ -178,68 +186,73 @@ public class MonitorTest { assertThat(p2) .isNotReady() .wasGracefullyTerminatedBefore(p1); + verify(fileSystem).reset(); } @Test public void stop_all_processes_if_monitor_shutdowns() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); HttpProcessClient p1 = new HttpProcessClient(tempDir, "p1"); HttpProcessClient p2 = new HttpProcessClient(tempDir, "p2"); - monitor.start(Arrays.asList(p1.newCommand(), p2.newCommand())); + underTest.start(Arrays.asList(p1.newCommand(), p2.newCommand())); assertThat(p1).isReady(); assertThat(p2).isReady(); // emulate CTRL-C - monitor.getShutdownHook().run(); - monitor.getShutdownHook().join(); + underTest.getShutdownHook().run(); + underTest.getShutdownHook().join(); assertThat(p1).wasGracefullyTerminated(); assertThat(p2).wasGracefullyTerminated(); + + verify(fileSystem).reset(); } @Test public void restart_all_processes_if_one_asks_for_restart() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); HttpProcessClient p1 = new HttpProcessClient(tempDir, "p1"); HttpProcessClient p2 = new HttpProcessClient(tempDir, "p2"); - monitor.start(Arrays.asList(p1.newCommand(), p2.newCommand())); + underTest.start(Arrays.asList(p1.newCommand(), p2.newCommand())); assertThat(p1).isReady(); assertThat(p2).isReady(); p2.restart(); - assertThat(monitor.waitForOneRestart()).isTrue(); + assertThat(underTest.waitForOneRestart()).isTrue(); assertThat(p1) - .wasStarted(2) - .wasGracefullyTerminated(1); + .wasStarted(2) + .wasGracefullyTerminated(1); assertThat(p2) - .wasStarted(2) - .wasGracefullyTerminated(1); + .wasStarted(2) + .wasGracefullyTerminated(1); - monitor.stop(); + underTest.stop(); assertThat(p1) - .wasStarted(2) - .wasGracefullyTerminated(2); + .wasStarted(2) + .wasGracefullyTerminated(2); assertThat(p2) - .wasStarted(2) - .wasGracefullyTerminated(2); + .wasStarted(2) + .wasGracefullyTerminated(2); + + verify(fileSystem, times(2)).reset(); } @Test public void stop_all_processes_if_one_shutdowns() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); HttpProcessClient p1 = new HttpProcessClient(tempDir, "p1"); HttpProcessClient p2 = new HttpProcessClient(tempDir, "p2"); - monitor.start(Arrays.asList(p1.newCommand(), p2.newCommand())); + underTest.start(Arrays.asList(p1.newCommand(), p2.newCommand())); assertThat(p1.isReady()).isTrue(); assertThat(p2.isReady()).isTrue(); // kill p1 -> waiting for detection by monitor than termination of p2 p1.kill(); - monitor.awaitTermination(); + underTest.awaitTermination(); assertThat(p1) .isNotReady() @@ -247,15 +260,17 @@ public class MonitorTest { assertThat(p2) .isNotReady() .wasGracefullyTerminated(); + + verify(fileSystem).reset(); } @Test public void stop_all_processes_if_one_fails_to_start() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); HttpProcessClient p1 = new HttpProcessClient(tempDir, "p1"); HttpProcessClient p2 = new HttpProcessClient(tempDir, "p2", -1); try { - monitor.start(Arrays.asList(p1.newCommand(), p2.newCommand())); + underTest.start(Arrays.asList(p1.newCommand(), p2.newCommand())); fail(); } catch (Exception expected) { assertThat(p1) @@ -283,13 +298,13 @@ public class MonitorTest { @Test public void fail_to_start_if_bad_class_name() throws Exception { - monitor = newDefaultMonitor(tempDir); + underTest = newDefaultMonitor(tempDir); JavaCommand command = new JavaCommand("test") .addClasspath(testJar.getAbsolutePath()) .setClassName("org.sonar.process.test.Unknown"); try { - monitor.start(Arrays.asList(command)); + underTest.start(singletonList(command)); fail(); } catch (Exception e) { // expected @@ -299,17 +314,28 @@ public class MonitorTest { @Test public void watchForHardStop_adds_a_hardStopWatcher_thread_and_starts_it() throws Exception { - Monitor monitor = newDefaultMonitor(tempDir); - assertThat(monitor.hardStopWatcher).isNull(); + underTest = newDefaultMonitor(tempDir, true); + assertThat(underTest.hardStopWatcher).isNull(); - monitor.watchForHardStop(); + HttpProcessClient p1 = new HttpProcessClient(tempDir, "p1"); + underTest.start(singletonList(p1.newCommand())); - assertThat(monitor.hardStopWatcher).isNotNull(); - assertThat(monitor.hardStopWatcher.isAlive()).isTrue(); + assertThat(underTest.hardStopWatcher).isNotNull(); + assertThat(underTest.hardStopWatcher.isAlive()).isTrue(); + + p1.kill(); + underTest.awaitTermination(); + + assertThat(underTest.hardStopWatcher.isAlive()).isFalse(); } private Monitor newDefaultMonitor(File tempDir) throws IOException { - return new Monitor(tempDir, exit); + return newDefaultMonitor(tempDir, false); + } + + private Monitor newDefaultMonitor(File tempDir, boolean watchForHardStop) throws IOException { + when(fileSystem.getTempDir()).thenReturn(tempDir); + return new Monitor(fileSystem, exit, watchForHardStop); } /** @@ -369,7 +395,7 @@ public class MonitorTest { public void restart() { try { HttpRequest httpRequest = HttpRequest.post("http://localhost:" + httpPort + "/" + "restart") - .readTimeout(5000).connectTimeout(5000); + .readTimeout(5000).connectTimeout(5000); if (!httpRequest.ok() || !"ok".equals(httpRequest.body())) { throw new IllegalStateException("Wrong response calling restart"); } diff --git a/sonar-application/src/main/java/org/sonar/application/App.java b/sonar-application/src/main/java/org/sonar/application/App.java index e09899b1fa4..ca2f98d62b8 100644 --- a/sonar-application/src/main/java/org/sonar/application/App.java +++ b/sonar-application/src/main/java/org/sonar/application/App.java @@ -39,8 +39,8 @@ public class App implements Stoppable { private final Monitor monitor; - public App(File tempDir) { - this(Monitor.create(tempDir)); + public App(AppFileSystem appFileSystem, boolean watchForHardStop) { + this(Monitor.create(appFileSystem, watchForHardStop)); } App(Monitor monitor) { @@ -48,14 +48,11 @@ public class App implements Stoppable { } public void start(Props props) { - if (props.valueAsBoolean(ProcessProperties.ENABLE_STOP_COMMAND, false)) { - monitor.watchForHardStop(); - } monitor.start(createCommands(props)); monitor.awaitTermination(); } - private List createCommands(Props props) { + private static List createCommands(Props props) { List commands = new ArrayList<>(); File homeDir = props.nonNullValueAsFile(ProcessProperties.PATH_HOME); JavaCommand elasticsearch = new JavaCommand("search"); @@ -102,11 +99,14 @@ public class App implements Stoppable { CommandLineParser cli = new CommandLineParser(); Properties rawProperties = cli.parseArguments(args); Props props = new PropsBuilder(rawProperties, new JdbcSettings()).build(); + AppFileSystem appFileSystem = new AppFileSystem(props); + appFileSystem.verifyProps(); AppLogging logging = new AppLogging(); logging.configure(props); - File tempDir = props.nonNullValueAsFile(ProcessProperties.PATH_TEMP); - App app = new App(tempDir); + // used by orchestrator + boolean watchForHardStop = props.valueAsBoolean(ProcessProperties.ENABLE_STOP_COMMAND, false); + App app = new App(appFileSystem, watchForHardStop); app.start(props); } diff --git a/sonar-application/src/main/java/org/sonar/application/AppFileSystem.java b/sonar-application/src/main/java/org/sonar/application/AppFileSystem.java new file mode 100644 index 00000000000..32f563737b2 --- /dev/null +++ b/sonar-application/src/main/java/org/sonar/application/AppFileSystem.java @@ -0,0 +1,120 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.application; + +import java.io.File; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.process.Props; +import org.sonar.process.monitor.FileSystem; + +import static org.apache.commons.io.FileUtils.deleteQuietly; +import static org.apache.commons.io.FileUtils.forceMkdir; +import static org.sonar.process.ProcessProperties.PATH_DATA; +import static org.sonar.process.ProcessProperties.PATH_HOME; +import static org.sonar.process.ProcessProperties.PATH_LOGS; +import static org.sonar.process.ProcessProperties.PATH_TEMP; +import static org.sonar.process.ProcessProperties.PATH_WEB; + +public class AppFileSystem implements FileSystem { + private static final Logger LOG = LoggerFactory.getLogger(AppFileSystem.class); + + private static final String DEFAULT_DATA_DIRECTORY_NAME = "data"; + private static final String DEFAULT_WEB_DIRECTORY_NAME = "web"; + private static final String DEFAULT_LOGS_DIRECTORY_NAME = "logs"; + private static final String DEFAULT_TEMP_DIRECTORY_NAME = "temp"; + + private final Props props; + private final File homeDir; + private boolean initialized = false; + + public AppFileSystem(Props props) { + this.props = props; + this.homeDir = props.nonNullValueAsFile(PATH_HOME); + } + + public void verifyProps() { + ensurePropertyIsAbsolutePath(props, PATH_DATA, DEFAULT_DATA_DIRECTORY_NAME); + ensurePropertyIsAbsolutePath(props, PATH_WEB, DEFAULT_WEB_DIRECTORY_NAME); + ensurePropertyIsAbsolutePath(props, PATH_LOGS, DEFAULT_LOGS_DIRECTORY_NAME); + ensurePropertyIsAbsolutePath(props, PATH_TEMP, DEFAULT_TEMP_DIRECTORY_NAME); + this.initialized = true; + } + + /** + * Must be called after {@link #verifyProps()} + */ + @Override + public void reset() throws IOException { + if (!initialized) { + throw new IllegalStateException("method verifyProps must be called first"); + } + ensureDirectoryExists(props, PATH_DATA); + ensureDirectoryExists(props, PATH_WEB); + ensureDirectoryExists(props, PATH_LOGS); + createOrCleanDirectory(props, PATH_TEMP); + } + + @Override + public File getTempDir() { + return props.nonNullValueAsFile(PATH_TEMP); + } + + private File ensurePropertyIsAbsolutePath(Props props, String propKey, String defaultRelativePath) { + String path = props.value(propKey, defaultRelativePath); + File d = new File(path); + if (!d.isAbsolute()) { + d = new File(homeDir, path); + LOG.trace("Overriding property {} from relative path '{}' to absolute path '{}'", path, d.getAbsolutePath()); + props.set(propKey, d.getAbsolutePath()); + } + return d; + } + + private static boolean ensureDirectoryExists(Props props, String propKey) throws IOException { + File dir = props.nonNullValueAsFile(propKey); + if (dir.exists()) { + ensureIsNotAFile(propKey, dir); + return false; + } else { + LOG.trace("forceMkdir {}", dir.getAbsolutePath()); + forceMkdir(dir); + ensureIsNotAFile(propKey, dir); + return true; + } + } + + private static void ensureIsNotAFile(String propKey, File dir) { + if (!dir.isDirectory()) { + throw new IllegalStateException(String.format("Property '%s' is not valid, not a directory: %s", + propKey, dir.getAbsolutePath())); + } + } + + private static void createOrCleanDirectory(Props props, String propKey) throws IOException { + File dir = props.nonNullValueAsFile(propKey); + LOG.info("Deleting and/or creating temp directory {}", dir.getAbsolutePath()); + if (!ensureDirectoryExists(props, propKey)) { + deleteQuietly(dir); + forceMkdir(dir); + } + } +} diff --git a/sonar-application/src/main/java/org/sonar/application/PropsBuilder.java b/sonar-application/src/main/java/org/sonar/application/PropsBuilder.java index d915e1609ad..1e2d25778e2 100644 --- a/sonar-application/src/main/java/org/sonar/application/PropsBuilder.java +++ b/sonar-application/src/main/java/org/sonar/application/PropsBuilder.java @@ -19,12 +19,6 @@ */ package org.sonar.application; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.sonar.process.ConfigurationUtils; -import org.sonar.process.ProcessProperties; -import org.sonar.process.Props; - import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -33,6 +27,10 @@ import java.io.Reader; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.Properties; +import org.apache.commons.io.IOUtils; +import org.sonar.process.ConfigurationUtils; +import org.sonar.process.ProcessProperties; +import org.sonar.process.Props; class PropsBuilder { @@ -51,8 +49,7 @@ class PropsBuilder { } /** - * Load optional conf/sonar.properties, interpolates environment variables and - * initializes file system + * Load optional conf/sonar.properties, interpolates environment variables */ Props build() throws IOException { Properties p = loadPropertiesFile(homeDir); @@ -66,12 +63,6 @@ class PropsBuilder { Props props = new Props(p); ProcessProperties.completeDefaults(props); - // init file system - initExistingDir(props, ProcessProperties.PATH_DATA, "data"); - initExistingDir(props, ProcessProperties.PATH_WEB, "web"); - initExistingDir(props, ProcessProperties.PATH_LOGS, "logs"); - initTempDir(props); - // check JDBC properties and set path to driver jdbcSettings.checkAndComplete(homeDir, props); @@ -96,31 +87,4 @@ class PropsBuilder { } return p; } - - private void initTempDir(Props props) throws IOException { - File dir = configureDir(props, ProcessProperties.PATH_TEMP, "temp"); - FileUtils.deleteQuietly(dir); - FileUtils.forceMkdir(dir); - } - - private void initExistingDir(Props props, String propKey, String defaultRelativePath) throws IOException { - File dir = configureDir(props, propKey, defaultRelativePath); - if (!dir.exists()) { - FileUtils.forceMkdir(dir); - } - if (!dir.isDirectory()) { - throw new IllegalStateException(String.format("Property '%s' is not valid, not a directory: %s", - propKey, dir.getAbsolutePath())); - } - } - - private File configureDir(Props props, String propKey, String defaultRelativePath) { - String path = props.value(propKey, defaultRelativePath); - File d = new File(path); - if (!d.isAbsolute()) { - d = new File(homeDir, path); - } - props.set(propKey, d.getAbsolutePath()); - return d; - } } diff --git a/sonar-application/src/test/java/org/sonar/application/AppFileSystemTest.java b/sonar-application/src/test/java/org/sonar/application/AppFileSystemTest.java new file mode 100644 index 00000000000..baac93fa203 --- /dev/null +++ b/sonar-application/src/test/java/org/sonar/application/AppFileSystemTest.java @@ -0,0 +1,202 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.application; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.sonar.process.Props; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AppFileSystemTest { + + private static final String PROPERTY_SONAR_PATH_WEB = "sonar.path.web"; + private static final String PROPERTY_SONAR_PATH_DATA = "sonar.path.data"; + private static final String PROPERTY_SONAR_PATH_LOGS = "sonar.path.logs"; + private static final String PROPERTY_SONAR_PATH_TEMP = "sonar.path.temp"; + private static final String NON_DEFAULT_DATA_DIR_NAME = "toto"; + private static final String NON_DEFAULT_WEB_DIR_NAME = "tutu"; + private static final String NON_DEFAULT_LOGS_DIR_NAME = "titi"; + private static final String NON_DEFAULT_TEMP_DIR_NAME = "tatta"; + private static final String DEFAULT_DATA_DIR_NAME = "data"; + private static final String DEFAULT_WEB_DIR_NAME = "web"; + private static final String DEFAULT_LOGS_DIR_NAME = "logs"; + private static final String DEFAULT_TEMP_DIR_NAME = "temp"; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private File homeDir; + private Properties properties; + + @Before + public void before() throws IOException { + homeDir = temp.newFolder(); + + properties = new Properties(); + properties.setProperty("sonar.path.home", homeDir.getAbsolutePath()); + } + + @Test + public void verifyProps_set_dir_path_absolute_based_on_home_dir_and_default_names_when_no_property() { + Props props = new Props(properties); + AppFileSystem underTest = new AppFileSystem(props); + + underTest.verifyProps(); + + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_DATA)).isEqualTo(new File(homeDir, DEFAULT_DATA_DIR_NAME).getAbsolutePath()); + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_WEB)).isEqualTo(new File(homeDir, DEFAULT_WEB_DIR_NAME).getAbsolutePath()); + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_LOGS)).isEqualTo(new File(homeDir, DEFAULT_LOGS_DIR_NAME).getAbsolutePath()); + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_TEMP)).isEqualTo(new File(homeDir, DEFAULT_TEMP_DIR_NAME).getAbsolutePath()); + } + + @Test + public void verifyProps_can_be_called_multiple_times() { + AppFileSystem underTest = new AppFileSystem(new Props(properties)); + + underTest.verifyProps(); + underTest.verifyProps(); + } + + @Test + public void reset_throws_ISE_if_verifyProps_not_called_first() throws Exception { + AppFileSystem underTest = new AppFileSystem(new Props(properties)); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("method verifyProps must be called first"); + + underTest.reset(); + } + + @Test + public void verifyProps_makes_dir_path_absolute_based_on_home_dir_when_relative() throws Exception { + properties.setProperty(PROPERTY_SONAR_PATH_WEB, NON_DEFAULT_WEB_DIR_NAME); + properties.setProperty(PROPERTY_SONAR_PATH_DATA, NON_DEFAULT_DATA_DIR_NAME); + properties.setProperty(PROPERTY_SONAR_PATH_LOGS, NON_DEFAULT_LOGS_DIR_NAME); + properties.setProperty(PROPERTY_SONAR_PATH_TEMP, NON_DEFAULT_TEMP_DIR_NAME); + + Props props = new Props(properties); + AppFileSystem underTest = new AppFileSystem(props); + + underTest.verifyProps(); + + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_DATA)).isEqualTo(new File(homeDir, NON_DEFAULT_DATA_DIR_NAME).getAbsolutePath()); + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_WEB)).isEqualTo(new File(homeDir, NON_DEFAULT_WEB_DIR_NAME).getAbsolutePath()); + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_LOGS)).isEqualTo(new File(homeDir, NON_DEFAULT_LOGS_DIR_NAME).getAbsolutePath()); + assertThat(props.nonNullValue(PROPERTY_SONAR_PATH_TEMP)).isEqualTo(new File(homeDir, NON_DEFAULT_TEMP_DIR_NAME).getAbsolutePath()); + } + + @Test + public void reset_creates_dir_all_dirs_if_they_don_t_exist() throws Exception { + AppFileSystem underTest = new AppFileSystem(new Props(properties)); + + underTest.verifyProps(); + + File dataDir = new File(homeDir, DEFAULT_DATA_DIR_NAME); + File webDir = new File(homeDir, DEFAULT_WEB_DIR_NAME); + File logsDir = new File(homeDir, DEFAULT_LOGS_DIR_NAME); + File tempDir = new File(homeDir, DEFAULT_TEMP_DIR_NAME); + assertThat(dataDir).doesNotExist(); + assertThat(webDir).doesNotExist(); + assertThat(logsDir).doesNotExist(); + assertThat(tempDir).doesNotExist(); + + underTest.reset(); + + assertThat(dataDir).exists().isDirectory(); + assertThat(webDir).exists().isDirectory(); + assertThat(logsDir).exists().isDirectory(); + assertThat(tempDir).exists().isDirectory(); + } + + @Test + public void reset_delete_temp_dir_if_already_exists() throws Exception { + File tempDir = new File(homeDir, DEFAULT_TEMP_DIR_NAME); + assertThat(tempDir.mkdir()).isTrue(); + File fileInTempDir = new File(tempDir, "someFile.txt"); + assertThat(fileInTempDir.createNewFile()).isTrue(); + + AppFileSystem underTest = new AppFileSystem(new Props(properties)); + underTest.verifyProps(); + underTest.reset(); + + assertThat(tempDir).exists(); + assertThat(fileInTempDir).doesNotExist(); + } + + @Test + public void reset_throws_ISE_if_data_dir_is_a_file() throws Exception { + resetThrowsISEIfDirIsAFile(PROPERTY_SONAR_PATH_DATA); + } + + @Test + public void reset_throws_ISE_if_web_dir_is_a_file() throws Exception { + resetThrowsISEIfDirIsAFile(PROPERTY_SONAR_PATH_WEB); + } + + @Test + public void reset_throws_ISE_if_logs_dir_is_a_file() throws Exception { + resetThrowsISEIfDirIsAFile(PROPERTY_SONAR_PATH_LOGS); + } + + @Test + public void reset_throws_ISE_if_temp_dir_is_a_file() throws Exception { + resetThrowsISEIfDirIsAFile(PROPERTY_SONAR_PATH_TEMP); + } + + private void resetThrowsISEIfDirIsAFile(String property) throws IOException { + File file = new File(homeDir, "zoom.store"); + assertThat(file.createNewFile()).isTrue(); + + properties.setProperty(property, "zoom.store"); + + AppFileSystem underTest = new AppFileSystem(new Props(properties)); + + underTest.verifyProps(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Property '" + property + "' is not valid, not a directory: " + file.getAbsolutePath()); + + underTest.reset(); + } + + // @Test +// public void fail_if_required_directory_is_a_file() throws Exception { +// // /data is missing +// FileUtils.forceMkdir(webDir); +// FileUtils.forceMkdir(logsDir); +// try { +// FileUtils.touch(dataDir); +// new PropsBuilder(new Properties(), jdbcSettings, homeDir).build(); +// fail(); +// } catch (IllegalStateException e) { +// assertThat(e.getMessage()).startsWith("Property 'sonar.path.data' is not valid, not a directory: " + dataDir.getAbsolutePath()); +// } +// } + +} diff --git a/sonar-application/src/test/java/org/sonar/application/PropsBuilderTest.java b/sonar-application/src/test/java/org/sonar/application/PropsBuilderTest.java index de5069940d2..ad5fb1ddb8a 100644 --- a/sonar-application/src/test/java/org/sonar/application/PropsBuilderTest.java +++ b/sonar-application/src/test/java/org/sonar/application/PropsBuilderTest.java @@ -19,6 +19,10 @@ */ package org.sonar.application; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Properties; import org.apache.commons.io.FileUtils; import org.junit.Before; import org.junit.Rule; @@ -26,13 +30,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.process.Props; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Properties; - import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; public class PropsBuilderTest { @@ -40,39 +38,21 @@ public class PropsBuilderTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); - File homeDir; - File dataDir; - File webDir; - File logsDir; - JdbcSettings jdbcSettings = mock(JdbcSettings.class); + private File homeDir; + private JdbcSettings jdbcSettings = mock(JdbcSettings.class); @Before public void before() throws IOException { homeDir = temp.newFolder(); - dataDir = new File(homeDir, "data"); - webDir = new File(homeDir, "web"); - logsDir = new File(homeDir, "logs"); } @Test public void build_props() throws Exception { - FileUtils.forceMkdir(dataDir); - FileUtils.forceMkdir(webDir); - FileUtils.forceMkdir(logsDir); Properties rawProperties = new Properties(); rawProperties.setProperty("foo", "bar"); Props props = new PropsBuilder(rawProperties, jdbcSettings, homeDir).build(); - assertThat(props.nonNullValueAsFile("sonar.path.logs")).isEqualTo(logsDir); - assertThat(props.nonNullValueAsFile("sonar.path.home")).isEqualTo(homeDir); - - // create /temp - File tempDir = props.nonNullValueAsFile("sonar.path.temp"); - assertThat(tempDir).isDirectory().exists(); - assertThat(tempDir.getName()).isEqualTo("temp"); - assertThat(tempDir.getParentFile()).isEqualTo(homeDir); - assertThat(props.value("foo")).isEqualTo("bar"); assertThat(props.value("unknown")).isNull(); @@ -80,37 +60,9 @@ public class PropsBuilderTest { assertThat(props.valueAsInt("sonar.search.port")).isEqualTo(9001); } - @Test - public void create_missing_required_directory() throws Exception { - // /data is missing - FileUtils.forceMkdir(webDir); - FileUtils.forceMkdir(logsDir); - - File dataDir = new File(homeDir, "data"); - new PropsBuilder(new Properties(), jdbcSettings, homeDir).build(); - assertThat(dataDir).isDirectory().exists(); - } - - @Test - public void fail_if_required_directory_is_a_file() throws Exception { - // /data is missing - FileUtils.forceMkdir(webDir); - FileUtils.forceMkdir(logsDir); - try { - FileUtils.touch(dataDir); - new PropsBuilder(new Properties(), jdbcSettings, homeDir).build(); - fail(); - } catch (IllegalStateException e) { - assertThat(e.getMessage()).startsWith("Property 'sonar.path.data' is not valid, not a directory: " + dataDir.getAbsolutePath()); - } - } - @Test public void load_properties_file_if_exists() throws Exception { FileUtils.write(new File(homeDir, "conf/sonar.properties"), "sonar.jdbc.username=angela\nsonar.origin=file"); - FileUtils.forceMkdir(dataDir); - FileUtils.forceMkdir(webDir); - FileUtils.forceMkdir(logsDir); Properties rawProperties = new Properties(); rawProperties.setProperty("sonar.origin", "raw"); @@ -132,10 +84,6 @@ public class PropsBuilderTest { @Test public void do_not_load_properties_file_if_not_exists() throws Exception { - FileUtils.forceMkdir(dataDir); - FileUtils.forceMkdir(webDir); - FileUtils.forceMkdir(logsDir); - Properties rawProperties = new Properties(); rawProperties.setProperty("sonar.origin", "raw"); Props props = new PropsBuilder(rawProperties, jdbcSettings, homeDir).build();