SONAR-9811 change es data dir to data/es5, delete data/es during startup

5 files changed, 109 insertions, 19 deletions

diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsFileSystem.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsFileSystem.java
index 73fc79edbe4..b7f87df7a76 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/es/EsFileSystem.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsFileSystem.java
@@ -20,7 +20,8 @@
 package org.sonar.application.es;
 
 import java.io.File;
-import org.apache.commons.lang.StringUtils;
+import java.util.Collections;
+import java.util.List;
 import org.sonar.process.ProcessProperties;
 import org.sonar.process.Props;
 
@@ -34,6 +35,7 @@ import org.sonar.process.Props;
  */
 public class EsFileSystem {
   private final File homeDirectory;
+  private final List<File> outdatedDataDirectories;
   private final File dataDirectory;
   private final File confDirectory;
   private final File logDirectory;
@@ -42,17 +44,20 @@ public class EsFileSystem {
     File sqHomeDir = props.nonNullValueAsFile(ProcessProperties.PATH_HOME);
 
     this.homeDirectory = new File(sqHomeDir, "elasticsearch");
-    this.dataDirectory = buildDataDir(props, sqHomeDir);
+    this.outdatedDataDirectories = buildOutdatedDataDirs(props);
+    this.dataDirectory = buildDataDir(props);
     this.confDirectory = buildConfDir(props);
     this.logDirectory = buildLogPath(props);
   }
 
-  private static File buildDataDir(Props props, File sqHomeDir) {
-    String dataPath = props.value(ProcessProperties.PATH_DATA);
-    if (StringUtils.isNotEmpty(dataPath)) {
-      return new File(dataPath, "es");
-    }
-    return new File(sqHomeDir, "data/es");
+  private static List<File> buildOutdatedDataDirs(Props props) {
+    String dataPath = props.nonNullValue(ProcessProperties.PATH_DATA);
+    return Collections.singletonList(new File(dataPath, "es"));
+  }
+
+  private static File buildDataDir(Props props) {
+    String dataPath = props.nonNullValue(ProcessProperties.PATH_DATA);
+    return new File(dataPath, "es5");
   }
 
   private static File buildLogPath(Props props) {
@@ -68,6 +73,10 @@ public class EsFileSystem {
     return homeDirectory;
   }
 
+  public List<File> getOutdatedDataDirectories() {
+    return Collections.unmodifiableList(outdatedDataDirectories);
+  }
+
   public File getDataDirectory() {
     return dataDirectory;
   }
diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java
index 1a8daf01ae3..0f172a9530f 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/process/ProcessLauncherImpl.java
@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.function.Supplier;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.application.command.AbstractCommand;
@@ -72,6 +73,7 @@ public class ProcessLauncherImpl implements ProcessLauncher {
   public ProcessMonitor launch(EsCommand esCommand) {
     Process process = null;
     try {
+      cleanupOutdatedEsData(esCommand);
       writeConfFiles(esCommand);
       ProcessBuilder processBuilder = create(esCommand);
       logLaunchedCommand(esCommand, processBuilder);
@@ -88,6 +90,20 @@ public class ProcessLauncherImpl implements ProcessLauncher {
     }
   }
 
+  private static void cleanupOutdatedEsData(EsCommand esCommand) {
+    EsFileSystem esFileSystem = esCommand.getFileSystem();
+    esFileSystem.getOutdatedDataDirectories().forEach(outdatedDir -> {
+      if (outdatedDir.exists()) {
+        LOG.info("Deleting outdated search index data directory {}", outdatedDir.getAbsolutePath());
+        try {
+          FileUtils.deleteDirectory(outdatedDir);
+        } catch (IOException e) {
+          LOG.info("Failed to delete outdated search index data directory " + outdatedDir.getAbsolutePath(), e);
+        }
+      }
+    });
+  }
+
   private static void writeConfFiles(EsCommand esCommand) {
     EsFileSystem esFileSystem = esCommand.getFileSystem();
     File confDir = esFileSystem.getConfDirectory();
diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsFileSystemTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsFileSystemTest.java
index e015b6bcce3..b2ae4c91cd1 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/es/EsFileSystemTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsFileSystemTest.java
@@ -51,6 +51,7 @@ public class EsFileSystemTest {
   @Test
   public void constructor_fails_with_IAE_if_temp_dir_property_is_not_defined() throws IOException {
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
 
     expectedException.expect(IllegalArgumentException.class);
@@ -60,29 +61,28 @@ public class EsFileSystemTest {
   }
 
   @Test
-  public void getHomeDirectory_is_elasticsearch_subdirectory_of_sq_home_directory() throws IOException {
-    File sqHomeDir = temp.newFolder();
+  public void constructor_fails_with_IAE_if_data_dir_property_is_not_defined() throws IOException {
     Props props = new Props(new Properties());
-    props.set(ProcessProperties.PATH_HOME, sqHomeDir.getAbsolutePath());
-    props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
-    props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
+    props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
 
-    EsFileSystem underTest = new EsFileSystem(props);
+    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expectMessage("Missing property: sonar.path.data");
 
-    assertThat(underTest.getHomeDirectory()).isEqualTo(new File(sqHomeDir, "elasticsearch"));
+    new EsFileSystem(props);
   }
 
   @Test
-  public void getDataDirectory_is_data_es_subdirectory_of_sq_home_directory_by_default() throws IOException {
+  public void getHomeDirectory_is_elasticsearch_subdirectory_of_sq_home_directory() throws IOException {
     File sqHomeDir = temp.newFolder();
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, sqHomeDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
 
     EsFileSystem underTest = new EsFileSystem(props);
 
-    assertThat(underTest.getDataDirectory()).isEqualTo(new File(sqHomeDir, "data/es"));
+    assertThat(underTest.getHomeDirectory()).isEqualTo(new File(sqHomeDir, "elasticsearch"));
   }
 
   @Test
@@ -99,7 +99,7 @@ public class EsFileSystemTest {
 
     EsFileSystem underTest = new EsFileSystem(props);
 
-    assertThat(underTest.getDataDirectory()).isEqualTo(new File(dataDir, "es"));
+    assertThat(underTest.getDataDirectory()).isEqualTo(new File(dataDir, "es5"));
   }
 
   @Test
@@ -107,6 +107,7 @@ public class EsFileSystemTest {
     File sqHomeDir = temp.newFolder();
     File logDir = temp.newFolder();
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, sqHomeDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, logDir.getAbsolutePath());
@@ -120,6 +121,7 @@ public class EsFileSystemTest {
   public void conf_directory_is_conf_es_subdirectory_of_sq_temp_directory() throws IOException {
     File tempDir = temp.newFolder();
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
@@ -133,6 +135,7 @@ public class EsFileSystemTest {
   public void getExecutable_resolve_executable_for_platform() throws IOException {
     File sqHomeDir = temp.newFolder();
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, sqHomeDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
@@ -150,6 +153,7 @@ public class EsFileSystemTest {
   public void getLog4j2Properties_is_in_es_conf_directory() throws IOException {
     File tempDir = temp.newFolder();
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
@@ -163,6 +167,7 @@ public class EsFileSystemTest {
   public void getElasticsearchYml_is_in_es_conf_directory() throws IOException {
     File tempDir = temp.newFolder();
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
@@ -176,6 +181,7 @@ public class EsFileSystemTest {
   public void getJvmOptions_is_in_es_conf_directory() throws IOException {
     File tempDir = temp.newFolder();
     Props props = new Props(new Properties());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_HOME, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, tempDir.getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java
index b6aad669aec..ff6db104e8b 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java
@@ -98,6 +98,7 @@ public class EsSettingsTest {
   private Props minimalProps() {
     Props props = new Props(new Properties());
     props.set(ProcessProperties.PATH_HOME, randomAlphanumeric(12));
+    props.set(ProcessProperties.PATH_DATA, randomAlphanumeric(12));
     props.set(ProcessProperties.PATH_TEMP, randomAlphanumeric(12));
     props.set(ProcessProperties.PATH_LOGS, randomAlphanumeric(12));
     props.set(CLUSTER_NAME, randomAlphanumeric(12));
@@ -111,6 +112,7 @@ public class EsSettingsTest {
     props.set(ProcessProperties.SEARCH_PORT, "1234");
     props.set(ProcessProperties.SEARCH_HOST, "127.0.0.1");
     props.set(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
     props.set(CLUSTER_NAME, "sonarqube");
@@ -147,6 +149,7 @@ public class EsSettingsTest {
     props.set(ProcessProperties.SEARCH_PORT, "1234");
     props.set(ProcessProperties.SEARCH_HOST, "127.0.0.1");
     props.set(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.CLUSTER_NAME, "sonarqube-1");
@@ -169,6 +172,7 @@ public class EsSettingsTest {
     props.set(ProcessProperties.SEARCH_PORT, "1234");
     props.set(ProcessProperties.SEARCH_HOST, "127.0.0.1");
     props.set(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
     EsSettings esSettings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE);
@@ -185,6 +189,7 @@ public class EsSettingsTest {
     props.set(ProcessProperties.SEARCH_PORT, "1234");
     props.set(ProcessProperties.SEARCH_HOST, "127.0.0.1");
     props.set(ProcessProperties.PATH_HOME, homeDir.getAbsolutePath());
+    props.set(ProcessProperties.PATH_DATA, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_TEMP, temp.newFolder().getAbsolutePath());
     props.set(ProcessProperties.PATH_LOGS, temp.newFolder().getAbsolutePath());
     EsSettings esSettings = new EsSettings(props, new EsFileSystem(props), System2.INSTANCE);
diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java
index 36a288c1f02..d98723eb892 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/process/ProcessLauncherImplTest.java
@@ -30,9 +30,14 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.process.ProcessId;
+import org.sonar.application.command.EsCommand;
+import org.sonar.application.command.EsJvmOptions;
 import org.sonar.application.command.JavaCommand;
 import org.sonar.application.command.JvmOptions;
+import org.sonar.application.es.EsFileSystem;
+import org.sonar.application.es.EsYmlSettings;
+import org.sonar.process.ProcessId;
+import org.sonar.process.Props;
 import org.sonar.process.sharedmemoryfile.AllProcessesCommands;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -112,6 +117,41 @@ public class ProcessLauncherImplTest {
   }
 
   @Test
+  public void clean_up_old_es_data() throws Exception {
+    File tempDir = temp.newFolder();
+    File homeDir = temp.newFolder();
+    File dataDir = temp.newFolder();
+    File logDir = temp.newFolder();
+    ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, () -> new TestProcessBuilder());
+    EsCommand command = createEsCommand(tempDir, homeDir, dataDir, logDir);
+
+    File outdatedEsDir = new File(dataDir, "es");
+    assertThat(outdatedEsDir.mkdir()).isTrue();
+    assertThat(outdatedEsDir.exists()).isTrue();
+
+    underTest.launch(command);
+
+    assertThat(outdatedEsDir.exists()).isFalse();
+  }
+
+  @Test
+  public void do_not_fail_if_outdated_es_directory_does_not_exist() throws Exception {
+    File tempDir = temp.newFolder();
+    File homeDir = temp.newFolder();
+    File dataDir = temp.newFolder();
+    File logDir = temp.newFolder();
+    ProcessLauncher underTest = new ProcessLauncherImpl(tempDir, commands, () -> new TestProcessBuilder());
+    EsCommand command = createEsCommand(tempDir, homeDir, dataDir, logDir);
+
+    File outdatedEsDir = new File(dataDir, "es");
+    assertThat(outdatedEsDir.exists()).isFalse();
+
+    underTest.launch(command);
+
+    assertThat(outdatedEsDir.exists()).isFalse();
+  }
+
+  @Test
   public void throw_ISE_if_command_fails() throws IOException {
     File tempDir = temp.newFolder();
     ProcessLauncherImpl.ProcessBuilder processBuilder = mock(ProcessLauncherImpl.ProcessBuilder.class, RETURNS_MOCKS);
@@ -124,6 +164,20 @@ public class ProcessLauncherImplTest {
     underTest.launch(new JavaCommand(ProcessId.ELASTICSEARCH, temp.newFolder()));
   }
 
+  private EsCommand createEsCommand(File tempDir, File homeDir, File dataDir, File logDir) throws IOException {
+    EsCommand command = new EsCommand(ProcessId.ELASTICSEARCH, temp.newFolder());
+    Props props = new Props(new Properties());
+    props.set("sonar.path.temp", tempDir.getAbsolutePath());
+    props.set("sonar.path.home", homeDir.getAbsolutePath());
+    props.set("sonar.path.data", dataDir.getAbsolutePath());
+    props.set("sonar.path.logs", logDir.getAbsolutePath());
+    command.setFileSystem(new EsFileSystem(props));
+    command.setEsYmlSettings(mock(EsYmlSettings.class));
+    command.setEsJvmOptions(mock(EsJvmOptions.class));
+    command.setLog4j2Properties(new Properties());
+    return command;
+  }
+
   private static class TestProcessBuilder implements ProcessLauncherImpl.ProcessBuilder {
     private List<String> commands = null;
     private File dir = null;