Move sonar-server, sonar-search and sonar-process into sub-dir server/

6 files changed, 554 insertions, 0 deletions

diff --git a/server/sonar-search/pom.xml b/server/sonar-search/pom.xml
new file mode 100644
index 00000000000..08db5991c1f
--- /dev/null
+++ b/server/sonar-search/pom.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <groupId>org.codehaus.sonar</groupId>
+    <artifactId>sonar-server-parent</artifactId>
+    <version>4.5-SNAPSHOT</version>
+    <relativePath>../</relativePath>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>sonar-search</artifactId>
+  <packaging>jar</packaging>
+  <name>SonarQube :: Search</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.codehaus.sonar</groupId>
+      <artifactId>sonar-process</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.elasticsearch</groupId>
+      <artifactId>elasticsearch</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.easytesting</groupId>
+      <artifactId>fest-assert</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>
\ No newline at end of file
diff --git a/server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java b/server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java
new file mode 100644
index 00000000000..4e00436ed35
--- /dev/null
+++ b/server/sonar-search/src/main/java/org/sonar/search/ElasticSearch.java
@@ -0,0 +1,125 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.search;
+
+import org.apache.commons.lang.StringUtils;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.process.Props;
+import org.sonar.search.script.ListUpdate;
+
+public class ElasticSearch extends org.sonar.process.Process {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearch.class);
+
+  public static final String ES_PORT_PROPERTY = "esPort";
+  public static final String ES_CLUSTER_PROPERTY = "esCluster";
+  public static final String ES_HOME_PROPERTY = "esHome";
+
+  public static final String MISSING_ES_PORT = "Missing ES port Argument";
+  public static final String MISSING_ES_HOME = "Missing ES home directory Argument";
+
+  public static final String DEFAULT_CLUSTER_NAME = "sonarqube";
+
+  private final Node node;
+
+  public ElasticSearch(Props props) {
+    super(props);
+
+
+    if (StringUtils.isEmpty(props.of(ES_HOME_PROPERTY, null))) {
+      throw new IllegalStateException(MISSING_ES_HOME);
+    }
+    String home = props.of(ES_HOME_PROPERTY);
+
+
+    if (StringUtils.isEmpty(props.of(ES_PORT_PROPERTY, null))) {
+      throw new IllegalStateException(MISSING_ES_PORT);
+    }
+    Integer port = props.intOf(ES_PORT_PROPERTY);
+
+
+    String clusterName = props.of(ES_CLUSTER_PROPERTY, DEFAULT_CLUSTER_NAME);
+
+    ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory());
+
+    ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder()
+      .put("index.merge.policy.max_merge_at_once", "200")
+      .put("index.merge.policy.segments_per_tier", "200")
+      .put("index.number_of_shards", "1")
+      .put("index.number_of_replicas", "0")
+      .put("index.store.type", "mmapfs")
+//
+      .put("indices.store.throttle.type", "merge")
+      .put("indices.store.throttle.max_bytes_per_sec", "200mb")
+//
+      .put("script.default_lang", "native")
+      .put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName())
+//
+      .put("cluster.name", clusterName)
+//
+      .put("node.name", "sonarqube-" + System.currentTimeMillis())
+      .put("node.data", true)
+      .put("node.local", false)
+//
+//      .put("network.bind_host", "127.0.0.1")
+      .put("http.enabled", false)
+//      .put("http.host", "127.0.0.1")
+
+      .put("transport.tcp.port", port)
+//      .put("http.port", 9200);
+
+      .put("path.home", home);
+
+    node = NodeBuilder.nodeBuilder()
+      .settings(esSettings)
+      .build().start();
+  }
+
+  @Override
+  public void execute() {
+    while (node != null && !node.isClosed()) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+    System.out.println("-- ES is done.");
+  }
+
+  public void shutdown() {
+    if (node != null) {
+      this.node.close();
+    }
+    super.shutdown();
+  }
+
+  public static void main(String... args) {
+    Props props = Props.create(System.getProperties());
+    ElasticSearch elasticSearch = new ElasticSearch(props);
+    elasticSearch.execute();
+  }
+}
\ No newline at end of file
diff --git a/server/sonar-search/src/main/java/org/sonar/search/NetworkUtils.java b/server/sonar-search/src/main/java/org/sonar/search/NetworkUtils.java
new file mode 100644
index 00000000000..6a78006c299
--- /dev/null
+++ b/server/sonar-search/src/main/java/org/sonar/search/NetworkUtils.java
@@ -0,0 +1,38 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.search;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+
+class NetworkUtils {
+
+  static int freePort() {
+    try {
+      return new ServerSocket(0).getLocalPort();
+    } catch (IOException e) {
+      throw new IllegalStateException("Can not find an open network port", e);
+    }
+  }
+
+  private static boolean isValidPort(int port) {
+    return port > 1023;
+  }
+}
diff --git a/server/sonar-search/src/main/java/org/sonar/search/script/ListUpdate.java b/server/sonar-search/src/main/java/org/sonar/search/script/ListUpdate.java
new file mode 100644
index 00000000000..d0acd6b3393
--- /dev/null
+++ b/server/sonar-search/src/main/java/org/sonar/search/script/ListUpdate.java
@@ -0,0 +1,147 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.search.script;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.script.AbstractExecutableScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.NativeScriptFactory;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * @since 4.4
+ */
+public class ListUpdate extends AbstractExecutableScript {
+
+  public static final String NAME = "listUpdate";
+
+  public static final String ID_FIELD = "idField";
+  public static final String ID_VALUE = "idValue";
+  public static final String FIELD = "field";
+  public static final String VALUE = "value";
+
+  public static class UpdateListScriptFactory implements NativeScriptFactory {
+    @Override
+    public ExecutableScript newScript(@Nullable Map<String, Object> params) {
+      String idField = XContentMapValues.nodeStringValue(params.get(ID_FIELD), null);
+      String idValue = XContentMapValues.nodeStringValue(params.get(ID_VALUE), null);
+      String field = XContentMapValues.nodeStringValue(params.get(FIELD), null);
+      Map value = null;
+      if (idField == null) {
+        throw new IllegalStateException("Missing '" + ID_FIELD + "' parameter");
+      }
+      if (idValue == null) {
+        throw new IllegalStateException("Missing '" + ID_VALUE + "' parameter");
+      }
+      if (field == null) {
+        throw new IllegalStateException("Missing '" + FIELD + "' parameter");
+      }
+
+      //NULL case is deletion of nested item
+      if (params.containsKey(VALUE)) {
+        Object obj = params.get(VALUE);
+        if (obj != null) {
+          value = XContentMapValues.nodeMapValue(params.get(VALUE), "Update item");
+        }
+      }
+
+      return new ListUpdate(idField, idValue, field, value);
+    }
+  }
+
+
+  private final String idField;
+  private final String idValue;
+  private final String field;
+  private final Map<String, Object> value;
+
+  private Map<String, Object> ctx;
+
+  public ListUpdate(String idField, String idValue, String field, Map<String, Object> value) {
+    this.idField = idField;
+    this.idValue = idValue;
+    this.field = field;
+    this.value = value;
+  }
+
+  @Override
+  public void setNextVar(String name, Object value) {
+    if (name.equals("ctx")) {
+      ctx = (Map<String, Object>) value;
+    }
+  }
+
+  @Override
+  public Object unwrap(Object value) {
+    return value;
+  }
+
+  @Override
+  public Object run() {
+    try {
+      //Get the Document's source from ctx
+      Map<String, Object> source = XContentMapValues.nodeMapValue(ctx.get("_source"), "source from context");
+
+      //Get the Object for list update
+      Object fieldValue = source.get(field);
+
+      if (fieldValue == null && value != null) {
+        // 0. The field does not exist (this is a upsert then)
+        source.put(field, value);
+      } else if (!XContentMapValues.isArray(fieldValue) && value != null) {
+        // 1. The field is not yet a list
+        Map currentFieldValue = XContentMapValues.nodeMapValue(fieldValue, "current FieldValue");
+        if (XContentMapValues.nodeStringValue(currentFieldValue.get(idField), null).equals(idValue)) {
+          source.put(field, value);
+        } else {
+          source.put(field, org.elasticsearch.common.collect.ImmutableSet.of(fieldValue, value));
+        }
+      } else {
+        // 3. field is a list
+        Collection items = (Collection) fieldValue;
+        Object target = null;
+        for (Object item : items) {
+          Map<String, Object> fields = (Map<String, Object>) item;
+          String itemIdValue = XContentMapValues.nodeStringValue(fields.get(idField), null);
+          if (itemIdValue != null && itemIdValue.equals(idValue)) {
+            target = item;
+            break;
+          }
+        }
+        if (target != null) {
+          items.remove(target);
+        }
+
+        //Supporting the update by NULL = deletion case
+        if (value != null) {
+          items.add(value);
+        }
+        source.put(field, items);
+      }
+    } catch (Exception e) {
+      throw new IllegalStateException("failed to execute listUpdate script", e);
+    }
+    return null;
+
+  }
+}
diff --git a/server/sonar-search/src/main/resources/logback.xml b/server/sonar-search/src/main/resources/logback.xml
new file mode 100644
index 00000000000..d873268fc12
--- /dev/null
+++ b/server/sonar-search/src/main/resources/logback.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!--
+  Configuration for default logger. Only used while embedded server is starting,
+  before proper logging configuration is loaded.
+
+  See http://logback.qos.ch/manual/configuration.html
+-->
+<configuration debug="false">
+  <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"/>
+
+  <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <File>${SONAR_HOME}/logs/sonar.log</File>
+    <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+      <param name="FileNamePattern" value="${SONAR_HOME}/logs/sonar.%i.log"/>
+      <param name="MinIndex" value="1"/>
+      <param name="MaxIndex" value="3"/>
+    </rollingPolicy>
+    <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+      <param name="MaxFileSize" value="5MB"/>
+    </triggeringPolicy>
+    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+      <!-- Use %d{yyyy.MM.dd HH:mm:ss.SSS} to display milliseconds -->
+      <pattern>
+        %d{yyyy.MM.dd HH:mm:ss} %-5level [%logger{20}] %X %msg%n
+      </pattern>
+    </encoder>
+  </appender>
+
+  <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+      <pattern>
+        %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n
+      </pattern>
+    </encoder>
+  </appender>
+
+  <logger name="console" additivity="false">
+    <level value="INFO"/>
+    <appender-ref ref="CONSOLE"/>
+  </logger>
+
+  <root>
+    <level value="INFO"/>
+    <appender-ref ref="CONSOLE"/>
+  </root>
+
+</configuration>
diff --git a/server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java b/server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java
new file mode 100644
index 00000000000..e48b95b6683
--- /dev/null
+++ b/server/sonar-search/src/test/java/org/sonar/search/ElasticSearchTest.java
@@ -0,0 +1,133 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.search;
+
+import org.apache.commons.io.FileUtils;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.process.Process;
+import org.sonar.process.Props;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.util.Properties;
+
+import static org.fest.assertions.Assertions.assertThat;
+import static org.junit.Assert.fail;
+
+public class ElasticSearchTest {
+
+  DatagramSocket socket;
+  File tempDirectory;
+  ElasticSearch elasticSearch;
+
+  @Before
+  public void setup() throws IOException {
+    socket = new DatagramSocket(0);
+    tempDirectory = FileUtils.getTempDirectory();
+    FileUtils.forceMkdir(tempDirectory);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    socket.close();
+    //TODO Fails on Jenkins
+    //FileUtils.deleteDirectory(tempDirectory);
+  }
+
+  @Test
+  public void missing_properties() {
+
+    Properties properties = new Properties();
+    properties.setProperty(Process.NAME_PROPERTY, "ES");
+    properties.setProperty(Process.HEARTBEAT_PROPERTY, Integer.toString(socket.getLocalPort()));
+
+    try {
+      new ElasticSearch(Props.create(properties));
+    } catch (Exception e) {
+      assertThat(e.getMessage()).isEqualTo(ElasticSearch.MISSING_ES_HOME);
+    }
+
+    properties.setProperty(ElasticSearch.ES_HOME_PROPERTY, tempDirectory.getAbsolutePath());
+    try {
+      new ElasticSearch(Props.create(properties));
+    } catch (Exception e) {
+      assertThat(e.getMessage()).isEqualTo(ElasticSearch.MISSING_ES_PORT);
+    }
+  }
+
+  @Test
+  public void can_connect() {
+
+    int port = NetworkUtils.freePort();
+
+    Properties properties = new Properties();
+    properties.setProperty(Process.NAME_PROPERTY, "ES");
+    properties.setProperty(Process.HEARTBEAT_PROPERTY, Integer.toString(socket.getLocalPort()));
+    properties.setProperty(ElasticSearch.ES_HOME_PROPERTY, tempDirectory.getAbsolutePath());
+    properties.setProperty(ElasticSearch.ES_PORT_PROPERTY, Integer.toString(port));
+
+    elasticSearch = new ElasticSearch(Props.create(properties));
+
+
+    // 0 assert that application is running
+    assertHeartBeat();
+
+    Settings settings = ImmutableSettings.settingsBuilder()
+      .put("cluster.name", "sonarqube")
+      .build();
+    TransportClient client = new TransportClient(settings)
+      .addTransportAddress(new InetSocketTransportAddress("localhost", port));
+
+    // 0 assert that we have a OK cluster available
+    assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN);
+
+    // 1 assert that we get a heartBeat from the application
+    assertHeartBeat();
+
+    // 2 assert that we can shut down ES
+    elasticSearch.shutdown();
+    try {
+      client.admin().cluster().prepareClusterStats().get().getStatus();
+      fail();
+    } catch (Exception e) {
+      assertThat(e.getMessage()).isEqualTo("No node available");
+    }
+  }
+
+  private void assertHeartBeat() {
+    DatagramPacket packet = new DatagramPacket(new byte[1024], 1024);
+    try {
+      socket.setSoTimeout(1200);
+      socket.receive(packet);
+    } catch (Exception e) {
+      throw new IllegalStateException("Did not get a heartbeat");
+    }
+    assertThat(packet.getData()).isNotNull();
+  }
+}
\ No newline at end of file