SONAR-3755 implement a disk-based map

7 files changed, 598 insertions, 2 deletions

diff --git a/sonar-batch/pom.xml b/sonar-batch/pom.xml
index d1b299715a0..2a3879298c7 100644
--- a/sonar-batch/pom.xml
+++ b/sonar-batch/pom.xml
@@ -13,6 +13,10 @@
 
   <dependencies>
     <dependency>
+      <groupId>com.akiban</groupId>
+      <artifactId>akiban-persistit</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.codehaus.sonar</groupId>
       <artifactId>sonar-core</artifactId>
     </dependency>
diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapContainer.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapContainer.java
index bd31a250e0f..a1c5d3be58d 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapContainer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapContainer.java
@@ -33,6 +33,7 @@ import org.sonar.batch.components.PastSnapshotFinderByDays;
 import org.sonar.batch.components.PastSnapshotFinderByPreviousAnalysis;
 import org.sonar.batch.components.PastSnapshotFinderByPreviousVersion;
 import org.sonar.batch.components.PastSnapshotFinderByVersion;
+import org.sonar.batch.index.Caches;
 import org.sonar.core.config.Logback;
 import org.sonar.core.i18n.I18nManager;
 import org.sonar.core.i18n.RuleI18nManager;
@@ -131,7 +132,8 @@ public class BootstrapContainer extends ComponentContainer {
       PastSnapshotFinderByPreviousVersion.class,
       PastMeasuresLoader.class,
       PastSnapshotFinder.class,
-      DefaultModelFinder.class
+      DefaultModelFinder.class,
+      Caches.class
     );
   }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/Cache.java b/sonar-batch/src/main/java/org/sonar/batch/index/Cache.java
new file mode 100644
index 00000000000..4bd6b3bf0a1
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/Cache.java
@@ -0,0 +1,220 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2012 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.batch.index;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.persistit.Exchange;
+import com.persistit.Key;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This cache is not thread-safe, due to direct usage of {@link com.persistit.Exchange}
+ */
+public class Cache<K, V extends Serializable> {
+
+  private static final String DEFAULT_GROUP = "_";
+  private final String name;
+  private final Exchange exchange;
+
+  Cache(String name, Exchange exchange) {
+    this.name = name;
+    this.exchange = exchange;
+  }
+
+  public Cache put(K key, V value) {
+    return put(DEFAULT_GROUP, key, value);
+  }
+
+  public Cache put(String group, K key, V value) {
+    try {
+      exchange.clear();
+      exchange.append(group).append(key);
+      exchange.getValue().put(value);
+      exchange.store();
+      return this;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to put element in cache", e);
+    }
+  }
+
+  /**
+   * Implements group-based retrieval of cache elements.
+   *
+   * @param key   The key.
+   * @param group The group.
+   * @return The element associated with key in the group, or null.
+   */
+  @SuppressWarnings("unchecked")
+  public V get(String group, K key) {
+    try {
+      exchange.clear();
+      exchange.append(group).append(key);
+      exchange.fetch();
+      if (!exchange.getValue().isDefined()) {
+        return null;
+      }
+      return (V) exchange.getValue().get();
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to get element from cache", e);
+    }
+  }
+
+
+  /**
+   * Returns the object associated with key from the cache, or null if not found.
+   *
+   * @param key The key whose associated value is to be retrieved.
+   * @return The value, or null if not found.
+   */
+  @SuppressWarnings("unchecked")
+  public V get(K key) {
+    return get(DEFAULT_GROUP, key);
+  }
+
+  public Cache remove(String group, K key) {
+    try {
+      exchange.clear();
+      exchange.append(group).append(key);
+      exchange.remove();
+      return this;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to get element from cache", e);
+    }
+  }
+
+  public Cache remove(K key) {
+    return remove(DEFAULT_GROUP, key);
+  }
+
+  /**
+   * Removes everything in the specified group.
+   *
+   * @param group The group name.
+   */
+  public Cache clear(String group) {
+    try {
+      exchange.clear();
+      exchange.append(group);
+      Key key = new Key(exchange.getKey());
+      key.to(Key.AFTER);
+      exchange.removeKeyRange(exchange.getKey(), key);
+      return this;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to clear cache group: " + group, e);
+    }
+  }
+
+
+  /**
+   * Removes everything in the default cache, but not any of the group caches.
+   */
+  public Cache clear() {
+    return clear(DEFAULT_GROUP);
+  }
+
+  /**
+   * Clears the default as well as all group caches.
+   */
+  public void clearAll() {
+    try {
+      exchange.clear();
+      exchange.removeAll();
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to clear cache", e);
+    }
+  }
+
+  /**
+   * Returns the set of cache keys associated with this group.
+   * TODO implement a lazy-loading equivalent with Iterator/Iterable
+   *
+   * @param group The group.
+   * @return The set of cache keys for this group.
+   */
+  @SuppressWarnings("unchecked")
+  public Set<K> keySet(String group) {
+    try {
+      Set<K> keys = Sets.newLinkedHashSet();
+      exchange.clear();
+      Exchange iteratorExchange = new Exchange(exchange);
+
+      iteratorExchange.append(group);
+      iteratorExchange.append(Key.BEFORE);
+      while (iteratorExchange.next(false)) {
+        keys.add((K) iteratorExchange.getKey().indexTo(-1).decode());
+      }
+      return keys;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to get cache keys", e);
+    }
+  }
+
+
+  /**
+   * Returns the set of keys associated with this cache.
+   *
+   * @return The set containing the keys for this cache.
+   */
+  public Set<K> keySet() {
+    return keySet(DEFAULT_GROUP);
+  }
+
+  // TODO implement a lazy-loading equivalent with Iterator/Iterable
+  public Collection<V> values(String group) {
+    try {
+      List<V> values = Lists.newLinkedList();
+      exchange.clear();
+      Exchange iteratorExchange = new Exchange(exchange);
+
+      iteratorExchange.append(group);
+      iteratorExchange.append(Key.BEFORE);
+      while (iteratorExchange.next(false)) {
+        values.add((V) iteratorExchange.getValue().get());
+      }
+      return values;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to get cache values", e);
+    }
+  }
+
+  public Iterable<V> values() {
+    return values(DEFAULT_GROUP);
+  }
+
+  public Collection<V> allValues() {
+    try {
+      List<V> values = Lists.newLinkedList();
+      exchange.clear();
+      Exchange iteratorExchange = new Exchange(exchange);
+      iteratorExchange.append(Key.BEFORE);
+      while (iteratorExchange.next(true)) {
+        values.add((V) iteratorExchange.getValue().get());
+      }
+      return values;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to get cache values", e);
+    }
+  }
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/Caches.java b/sonar-batch/src/main/java/org/sonar/batch/index/Caches.java
new file mode 100644
index 00000000000..f9e92456b2a
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/Caches.java
@@ -0,0 +1,104 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2012 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.batch.index;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.google.common.io.Files;
+import com.persistit.Exchange;
+import com.persistit.Persistit;
+import com.persistit.exception.PersistitException;
+import com.persistit.logging.Slf4jAdapter;
+import org.apache.commons.io.FileUtils;
+import org.picocontainer.Startable;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.BatchComponent;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * Factory of caches
+ *
+ * @since 3.6
+ */
+public class Caches implements BatchComponent, Startable {
+
+  private final Set<String> cacheNames = Sets.newHashSet();
+  private File tempDir;
+  private Persistit persistit;
+
+  public <K extends Serializable, V extends Serializable> Cache<K, V> createCache(String cacheName) {
+    Preconditions.checkState(!cacheNames.contains(cacheName), "Cache is already created: " + cacheName);
+
+    try {
+      Exchange exchange = persistit.getExchange("sonar-scan", cacheName, true);
+      Cache<K, V> cache = new Cache<K, V>(cacheName, exchange);
+      cacheNames.add(cacheName);
+      return cache;
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to create cache: " + cacheName, e);
+    }
+  }
+
+  @Override
+  public void start() {
+    try {
+      tempDir = Files.createTempDir();
+      persistit = new Persistit();
+      persistit.setPersistitLogger(new Slf4jAdapter(LoggerFactory.getLogger("PERSISTIT")));
+      Properties props = new Properties();
+      props.setProperty("datapath", tempDir.getAbsolutePath());
+      props.setProperty("buffer.count.8192", "10");
+      props.setProperty("logfile", "${datapath}/persistit.log");
+      props.setProperty("volume.1", "${datapath}/sonar-scan,create,pageSize:8192,initialSize:1M,extensionSize:1M,maximumSize:10G");
+      props.setProperty("journalpath", "${datapath}/journal");
+      persistit.setProperties(props);
+      persistit.initialize();
+    } catch (Exception e) {
+      throw new IllegalStateException("Fail to start caches", e);
+    }
+  }
+
+  @Override
+  public void stop() {
+    if (persistit != null) {
+      try {
+        persistit.close(false);
+        persistit = null;
+      } catch (PersistitException e) {
+        throw new IllegalStateException("Fail to close caches", e);
+      }
+    }
+    FileUtils.deleteQuietly(tempDir);
+    tempDir = null;
+    cacheNames.clear();
+  }
+
+  File tempDir() {
+    return tempDir;
+  }
+
+  Persistit persistit() {
+    return persistit;
+  }
+}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/CacheTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/CacheTest.java
index 6abf8691e29..975db7affe1 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/index/CacheTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/index/CacheTest.java
@@ -19,12 +19,127 @@
  */
 package org.sonar.batch.index;
 
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
+import static org.fest.assertions.Assertions.assertThat;
+
 public class CacheTest {
+  Caches caches = new Caches();
+
+  @Before
+  public void start() {
+    caches.start();
+  }
+
+  @After
+  public void stop() {
+    caches.stop();
+  }
+
+  @Test
+  public void test_put_get_remove() throws Exception {
+    Cache<String, String> cache = caches.createCache("issues");
+    assertThat(cache.get("foo")).isNull();
+    cache.put("foo", "bar");
+    assertThat(cache.get("foo")).isEqualTo("bar");
+    assertThat(cache.keySet()).containsOnly("foo");
+    cache.remove("foo");
+    assertThat(cache.get("foo")).isNull();
+    assertThat(cache.keySet()).isEmpty();
+  }
+
+  @Test
+  public void test_put_get_remove_on_groups() throws Exception {
+    Cache<String, Float> cache = caches.createCache("measures");
+    String group = "org/apache/struts/Action.java";
+    assertThat(cache.get(group, "ncloc")).isNull();
+    cache.put(group, "ncloc", 123f);
+    assertThat(cache.get(group, "ncloc")).isEqualTo(123f);
+    assertThat(cache.keySet(group)).containsOnly("ncloc");
+    assertThat(cache.get("ncloc")).isNull();
+    assertThat(cache.get(group)).isNull();
+    cache.remove(group, "ncloc");
+    assertThat(cache.get(group, "ncloc")).isNull();
+    assertThat(cache.keySet(group)).isEmpty();
+  }
+
+  @Test
+  public void test_clear_group() throws Exception {
+    Cache<String, Float> cache = caches.createCache("measures");
+    String group = "org/apache/struts/Action.java";
+    cache.put(group, "ncloc", 123f);
+    cache.put(group, "lines", 200f);
+    assertThat(cache.get(group, "lines")).isNotNull();
+
+    cache.clear("other group");
+    assertThat(cache.get(group, "lines")).isNotNull();
+
+    cache.clear(group);
+    assertThat(cache.get(group, "lines")).isNull();
+  }
+
+  @Test
+  public void test_operations_on_empty_cache() throws Exception {
+    Cache<String, String> cache = caches.createCache("issues");
+    assertThat(cache.get("foo")).isNull();
+    assertThat(cache.get("group", "foo")).isNull();
+    assertThat(cache.keySet()).isEmpty();
+    assertThat(cache.keySet("group")).isEmpty();
+    assertThat(cache.values()).isEmpty();
+    assertThat(cache.values("group")).isEmpty();
+
+    // do not fail
+    cache.remove("foo");
+    cache.remove("group", "foo");
+    cache.clear();
+    cache.clear("group");
+    cache.clearAll();
+  }
+
+  @Test
+  public void test_get_missing_key() {
+    Cache<String, String> cache = caches.createCache("issues");
+    assertThat(cache.get("foo")).isNull();
+  }
+
   @Test
-  public void test_put() throws Exception {
+  public void test_keyset_of_group() {
+    Cache<String, Float> cache = caches.createCache("issues");
+    cache.put("org/apache/struts/Action.java", "ncloc", 123f);
+    cache.put("org/apache/struts/Action.java", "lines", 200f);
+    cache.put("org/apache/struts/Filter.java", "coverage", 500f);
+    assertThat(cache.keySet("org/apache/struts/Action.java")).containsOnly("ncloc", "lines");
+    assertThat(cache.keySet("org/apache/struts/Filter.java")).containsOnly("coverage");
+  }
+
+  @Test
+  public void test_values_of_group() {
+    Cache<String, Float> cache = caches.createCache("issues");
+    cache.put("org/apache/struts/Action.java", "ncloc", 123f);
+    cache.put("org/apache/struts/Action.java", "lines", 200f);
+    cache.put("org/apache/struts/Filter.java", "lines", 500f);
+    assertThat(cache.values("org/apache/struts/Action.java")).containsOnly(123f, 200f);
+    assertThat(cache.values("org/apache/struts/Filter.java")).containsOnly(500f);
+  }
 
+  @Test
+  public void test_values() {
+    Cache<String, Float> cache = caches.createCache("issues");
+    cache.put("ncloc", 123f);
+    cache.put("lines", 200f);
+    assertThat(cache.values()).containsOnly(123f, 200f);
   }
 
+  @Test
+  public void test_all_values() {
+    Cache<String, Float> cache = caches.createCache("issues");
+    cache.put("org/apache/struts/Action.java", "ncloc", 123f);
+    cache.put("org/apache/struts/Action.java", "lines", 200f);
+    cache.put("org/apache/struts/Filter.java", "ncloc", 400f);
+    cache.put("org/apache/struts/Filter.java", "lines", 500f);
+
+    assertThat(cache.allValues()).containsOnly(123f, 200f, 400f, 500f);
+  }
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/CachesTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/CachesTest.java
new file mode 100644
index 00000000000..2471bb76c71
--- /dev/null
+++ b/sonar-batch/src/test/java/org/sonar/batch/index/CachesTest.java
@@ -0,0 +1,80 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2012 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.batch.index;
+
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.Serializable;
+
+import static org.fest.assertions.Assertions.assertThat;
+import static org.fest.assertions.Fail.fail;
+
+public class CachesTest {
+  Caches caches = new Caches();
+
+  @After
+  public void stop() {
+    caches.stop();
+  }
+
+  @Test
+  public void should_start_and_stop_persistit() throws Exception {
+    assertThat(caches.tempDir()).isNull();
+    assertThat(caches.persistit()).isNull();
+
+    caches.start();
+
+    File tempDir = caches.tempDir();
+    assertThat(tempDir).isDirectory().exists();
+    assertThat(caches.persistit()).isNotNull();
+    assertThat(caches.persistit().isInitialized()).isTrue();
+
+    caches.stop();
+
+    assertThat(tempDir).doesNotExist();
+    assertThat(caches.tempDir()).isNull();
+    assertThat(caches.persistit()).isNull();
+  }
+
+  @Test
+  public void should_create_cache() throws Exception {
+    caches.start();
+    Cache<String, Element> cache = caches.createCache("foo");
+    assertThat(cache).isNotNull();
+  }
+
+  @Test
+  public void should_not_create_cache_twice() throws Exception {
+    caches.start();
+    caches.<String, Element>createCache("foo");
+    try {
+      caches.<String, Element>createCache("foo");
+      fail();
+    } catch (IllegalStateException e) {
+      // ok
+    }
+  }
+
+  static class Element implements Serializable {
+
+  }
+}
diff --git a/sonar-batch/src/test/java/org/sonar/batch/issue/IssueCacheTest.java b/sonar-batch/src/test/java/org/sonar/batch/issue/IssueCacheTest.java
new file mode 100644
index 00000000000..ce6590dcba5
--- /dev/null
+++ b/sonar-batch/src/test/java/org/sonar/batch/issue/IssueCacheTest.java
@@ -0,0 +1,71 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2012 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.batch.issue;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.api.issue.Issue;
+import org.sonar.batch.index.Caches;
+import org.sonar.core.issue.DefaultIssue;
+
+import javax.annotation.Nullable;
+import java.util.Collection;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class IssueCacheTest {
+
+  Caches caches = new Caches();
+
+  @Before
+  public void start() {
+    caches.start();
+  }
+
+  @After
+  public void stop() {
+    caches.stop();
+  }
+
+  @Test
+  public void should_cache_issues() throws Exception {
+    IssueCache cache = new IssueCache();
+    DefaultIssue issue1 = new DefaultIssue().setKey("111").setComponentKey("org.struts.Action");
+    DefaultIssue issue2 = new DefaultIssue().setKey("222").setComponentKey("org.struts.Action");
+    DefaultIssue issue3 = new DefaultIssue().setKey("333").setComponentKey("org.struts.Filter");
+    cache.add(issue1).add(issue2).add(issue3);
+
+    assertThat(issueKeys(cache.componentIssues("org.struts.Action"))).containsOnly("111", "222");
+    assertThat(issueKeys(cache.componentIssues("org.struts.Filter"))).containsOnly("333");
+    assertThat(issueKeys(cache.issues())).containsOnly("111", "222", "333");
+  }
+
+  Collection<String> issueKeys(Collection<Issue> issues) {
+    return Collections2.transform(issues, new Function<Issue, String>() {
+      @Override
+      public String apply(@Nullable Issue issue) {
+        return issue.key();
+      }
+    });
+  }
+}