SONAR-6577 Offline mode in preview mode

6 files changed, 538 insertions, 12 deletions

diff --git a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java
index 63a1168639d..3b681edcff1 100644
--- a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java
+++ b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java
@@ -19,13 +19,13 @@
  */
 package org.sonar.home.cache;
 
-import org.apache.commons.io.FileUtils;
 import org.sonar.home.log.Log;
 
 import javax.annotation.CheckForNull;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 
 /**
  * This class is responsible for managing Sonar batch file cache. You can put file into cache and
@@ -108,7 +108,7 @@ public class FileCache {
       log.warn(String.format("Unable to rename %s to %s", sourceFile.getAbsolutePath(), targetFile.getAbsolutePath()));
       log.warn(String.format("A copy/delete will be tempted but with no garantee of atomicity"));
       try {
-        FileUtils.moveFile(sourceFile, targetFile);
+        Files.move(sourceFile.toPath(), targetFile.toPath());
       } catch (IOException e) {
         throw new IllegalStateException("Fail to move " + sourceFile.getAbsolutePath() + " to " + targetFile, e);
       }
@@ -121,7 +121,7 @@ public class FileCache {
 
   private void mkdirQuietly(File hashDir) {
     try {
-      FileUtils.forceMkdir(hashDir);
+      Files.createDirectories(hashDir.toPath());
     } catch (IOException e) {
       throw new IllegalStateException("Fail to create cache directory: " + hashDir, e);
     }
@@ -151,7 +151,7 @@ public class FileCache {
     if (!dir.isDirectory() || !dir.exists()) {
       log.debug("Create : " + dir.getAbsolutePath());
       try {
-        FileUtils.forceMkdir(dir);
+        Files.createDirectories(dir.toPath());
       } catch (IOException e) {
         throw new IllegalStateException("Unable to create " + debugTitle + dir.getAbsolutePath(), e);
       }
diff --git a/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java b/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java
index 18a7a5fe912..549f8d10f63 100644
--- a/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java
+++ b/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java
@@ -19,8 +19,6 @@
  */
 package org.sonar.home.cache;
 
-import org.apache.commons.io.IOUtils;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -49,16 +47,12 @@ public class FileHashes {
    * Computes the hash of given stream. The stream is closed by this method.
    */
   public String of(InputStream input) {
-    try {
+    try(InputStream is = input) {
       MessageDigest digest = MessageDigest.getInstance("MD5");
-      byte[] hash = digest(input, digest);
+      byte[] hash = digest(is, digest);
       return toHex(hash);
-
     } catch (Exception e) {
       throw new IllegalStateException("Fail to compute hash", e);
-
-    } finally {
-      IOUtils.closeQuietly(input);
     }
   }
 
diff --git a/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java
new file mode 100644
index 00000000000..f5493d3d6a0
--- /dev/null
+++ b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java
@@ -0,0 +1,265 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.home.cache;
+
+import org.sonar.home.log.Log;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.concurrent.Callable;
+
+import static java.nio.file.StandardOpenOption.*;
+
+public class PersistentCache {
+  private static final Charset ENCODING = StandardCharsets.UTF_8;
+  private static final String DIGEST_ALGO = "MD5";
+  private static final String LOCK_FNAME = ".lock";
+
+  private Path baseDir;
+
+  // eviction strategy is to expire entries after modification once a time duration has elapsed
+  private final long defaultDurationToExpireMs;
+  private final Log log;
+  private final boolean forceUpdate;
+
+  public PersistentCache(Path baseDir, long defaultDurationToExpireMs, Log log, boolean forceUpdate) {
+    this.baseDir = baseDir;
+    this.defaultDurationToExpireMs = defaultDurationToExpireMs;
+    this.log = log;
+    this.forceUpdate = forceUpdate;
+
+    log.info("cache: " + baseDir + " default expiration time (ms): " + defaultDurationToExpireMs);
+
+    if (forceUpdate) {
+      log.debug("cache: forcing update");
+    }
+
+    try {
+      Files.createDirectories(baseDir);
+    } catch (IOException e) {
+      throw new IllegalStateException("failed to create cache dir", e);
+    }
+  }
+
+  public Path getBaseDirectory() {
+    return baseDir;
+  }
+
+  public boolean isForceUpdate() {
+    return forceUpdate;
+  }
+
+  @CheckForNull
+  public synchronized String getString(@Nonnull String obj, @Nullable final Callable<String> valueLoader) throws Exception {
+    byte[] cached = get(obj, new Callable<byte[]>() {
+      @Override
+      public byte[] call() throws Exception {
+        String s = valueLoader.call();
+        if (s != null) {
+          return s.getBytes(ENCODING);
+        }
+        return null;
+      }
+    });
+
+    if (cached == null) {
+      return null;
+    }
+
+    return new String(cached, ENCODING);
+  }
+
+  @CheckForNull
+  public synchronized byte[] get(@Nonnull String obj, @Nullable Callable<byte[]> valueLoader) throws Exception {
+    String key = getKey(obj);
+    log.debug("cache: " + obj + " -> " + key);
+
+    try (FileLock l = lock()) {
+      if (!forceUpdate) {
+        byte[] cached = getCache(key);
+
+        if (cached != null) {
+          log.debug("cache hit for " + obj);
+          return cached;
+        }
+
+        log.debug("cache miss for " + obj);
+      } else {
+        log.debug("cache force update for " + obj);
+      }
+
+      if (valueLoader != null) {
+        byte[] value = valueLoader.call();
+        if (value != null) {
+          putCache(key, value);
+        }
+        return value;
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Deletes all cache entries
+   */
+  public synchronized void clear() {
+    log.info("cache: clearing");
+    try (FileLock l = lock()) {
+      deleteCacheEntries(createClearFilter());
+    } catch (IOException e) {
+      log.error("Error clearing cache", e);
+    }
+  }
+
+  /**
+   * Deletes cache entries that are no longer valid according to the default expiration time period.
+   */
+  public synchronized void clean() {
+    log.info("cache: cleaning");
+    try (FileLock l = lock()) {
+      deleteCacheEntries(createCleanFilter());
+    } catch (IOException e) {
+      log.error("Error cleaning cache", e);
+    }
+  }
+
+  private FileLock lock() throws IOException {
+    FileChannel ch = FileChannel.open(getLockPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+    return ch.lock();
+  }
+
+  private String getKey(String uri) {
+    try {
+      MessageDigest digest = MessageDigest.getInstance(DIGEST_ALGO);
+      digest.update(uri.getBytes(StandardCharsets.UTF_8));
+      return byteArrayToHex(digest.digest());
+    } catch (NoSuchAlgorithmException e) {
+      throw new IllegalStateException("Couldn't create hash", e);
+    }
+  }
+
+  private void deleteCacheEntries(DirectoryStream.Filter<Path> filter) throws IOException {
+    try (DirectoryStream<Path> stream = Files.newDirectoryStream(baseDir, filter)) {
+      for (Path p : stream) {
+        try {
+          Files.delete(p);
+        } catch (Exception e) {
+          log.error("Error deleting " + p, e);
+        }
+      }
+    }
+  }
+
+  private DirectoryStream.Filter<Path> createClearFilter() throws IOException {
+    return new DirectoryStream.Filter<Path>() {
+      @Override
+      public boolean accept(Path entry) throws IOException {
+        return !LOCK_FNAME.equals(entry.getFileName().toString());
+      }
+    };
+  }
+
+  private DirectoryStream.Filter<Path> createCleanFilter() throws IOException {
+    return new DirectoryStream.Filter<Path>() {
+      @Override
+      public boolean accept(Path entry) throws IOException {
+        if (LOCK_FNAME.equals(entry.getFileName().toString())) {
+          return false;
+        }
+
+        return isCacheEntryExpired(entry, PersistentCache.this.defaultDurationToExpireMs);
+      }
+    };
+  }
+
+  private void putCache(String key, byte[] value) throws UnsupportedEncodingException, IOException {
+    Path cachePath = getCacheEntryPath(key);
+    Files.write(cachePath, value, CREATE, WRITE, TRUNCATE_EXISTING);
+  }
+
+  private byte[] getCache(String key) throws IOException {
+    Path cachePath = getCacheEntryPath(key);
+
+    if (!validateCacheEntry(cachePath, this.defaultDurationToExpireMs)) {
+      return null;
+    }
+
+    return Files.readAllBytes(cachePath);
+  }
+
+  private boolean validateCacheEntry(Path cacheEntryPath, long durationToExpireMs) throws IOException {
+    if (!Files.exists(cacheEntryPath)) {
+      return false;
+    }
+
+    if (isCacheEntryExpired(cacheEntryPath, durationToExpireMs)) {
+      log.debug("cache: expiring entry");
+      Files.delete(cacheEntryPath);
+      return false;
+    }
+
+    return true;
+  }
+
+  private boolean isCacheEntryExpired(Path cacheEntryPath, long durationToExpireMs) throws IOException {
+    BasicFileAttributes attr = Files.readAttributes(cacheEntryPath, BasicFileAttributes.class);
+    long modTime = attr.lastModifiedTime().toMillis();
+
+    long age = System.currentTimeMillis() - modTime;
+
+    if (age > durationToExpireMs) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private Path getLockPath() {
+    return baseDir.resolve(LOCK_FNAME);
+  }
+
+  private Path getCacheEntryPath(String key) {
+    return baseDir.resolve(key);
+  }
+
+  private static String byteArrayToHex(byte[] a) {
+    StringBuilder sb = new StringBuilder(a.length * 2);
+    for (byte b : a) {
+      sb.append(String.format("%02x", b & 0xff));
+    }
+    return sb.toString();
+  }
+}
diff --git a/sonar-home/src/main/java/org/sonar/home/cache/PersistentCacheBuilder.java b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCacheBuilder.java
new file mode 100644
index 00000000000..c8fcf06d4d0
--- /dev/null
+++ b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCacheBuilder.java
@@ -0,0 +1,73 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.home.cache;
+
+import org.sonar.home.log.StandardLog;
+
+import org.sonar.home.log.Log;
+
+import javax.annotation.Nullable;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
+
+public class PersistentCacheBuilder {
+  private boolean forceUpdate = false;
+  private Path cachePath = null;
+  private Log log = new StandardLog();
+  private String name = "ws_cache";
+
+  public PersistentCache build() {
+    if (cachePath == null) {
+      setSonarHome(findHome());
+    }
+
+    return new PersistentCache(cachePath, TimeUnit.MILLISECONDS.convert(1L, TimeUnit.DAYS), log, forceUpdate);
+  }
+
+  public PersistentCacheBuilder setLog(Log log) {
+    this.log = log;
+    return this;
+  }
+
+  public PersistentCacheBuilder setSonarHome(@Nullable Path p) {
+    if (p != null) {
+      this.cachePath = p.resolve(name);
+    }
+    return this;
+  }
+
+  public PersistentCacheBuilder forceUpdate(boolean update) {
+    this.forceUpdate = update;
+    return this;
+  }
+
+  private static Path findHome() {
+    String home = System.getenv("SONAR_USER_HOME");
+
+    if (home != null) {
+      return Paths.get(home);
+    }
+
+    home = System.getProperty("user.home");
+    return Paths.get(home, ".sonar");
+  }
+}
diff --git a/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheBuilderTest.java b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheBuilderTest.java
new file mode 100644
index 00000000000..027e2d68924
--- /dev/null
+++ b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheBuilderTest.java
@@ -0,0 +1,62 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.home.cache;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.file.Files;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+public class PersistentCacheBuilderTest {
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  @Test
+  public void user_home_property_can_be_null() {
+    PersistentCache cache = new PersistentCacheBuilder().setSonarHome(null).build();
+    assertTrue(Files.isDirectory(cache.getBaseDirectory()));
+    assertThat(cache.getBaseDirectory().getFileName().toString()).isEqualTo("ws_cache");
+  }
+
+  @Test
+  public void set_user_home() {
+    PersistentCache cache = new PersistentCacheBuilder().setSonarHome(temp.getRoot().toPath()).build();
+
+    assertThat(cache.getBaseDirectory().getParent().toString()).isEqualTo(temp.getRoot().toPath().toString());
+    assertTrue(Files.isDirectory(cache.getBaseDirectory()));
+  }
+
+  @Test
+  public void read_system_env() {
+    System.setProperty("user.home", temp.getRoot().getAbsolutePath());
+
+    PersistentCache cache = new PersistentCacheBuilder().build();
+    assertTrue(Files.isDirectory(cache.getBaseDirectory()));
+    assertThat(cache.getBaseDirectory().getFileName().toString()).isEqualTo("ws_cache");
+
+    String expectedSonarHome = temp.getRoot().toPath().resolve(".sonar").toString();
+    assertThat(cache.getBaseDirectory().getParent().toString()).isEqualTo(expectedSonarHome);
+  }
+}
diff --git a/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheTest.java b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheTest.java
new file mode 100644
index 00000000000..8127e41885d
--- /dev/null
+++ b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheTest.java
@@ -0,0 +1,132 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.home.cache;
+
+import org.sonar.home.log.Slf4jLog;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.concurrent.Callable;
+
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.mock;
+import static org.assertj.core.api.Assertions.assertThat;
+import org.junit.Before;
+import org.junit.Test;
+
+public class PersistentCacheTest {
+  private final static String URI = "key1";
+  private final static String VALUE = "cache content";
+  private final static long CACHE_EXPIRE = 1000;
+  private PersistentCache cache = null;
+
+  @Rule
+  public TemporaryFolder tmp = new TemporaryFolder();
+
+  private Slf4jLog log = new Slf4jLog(FileCacheTest.class);
+
+  @Before
+  public void setUp() {
+    cache = new PersistentCache(tmp.getRoot().toPath(), CACHE_EXPIRE, log, false);
+  }
+
+  @Test
+  public void testCacheMiss() throws Exception {
+    assertCacheHit(false);
+  }
+
+  @Test
+  public void testNullLoader() throws Exception {
+    assertThat(cache.get(URI, null)).isNull();
+    assertCacheHit(false);
+  }
+
+  @Test
+  public void testNullValue() throws Exception {
+    // mocks have their methods returning null by default
+    Callable<byte[]> c = mock(Callable.class);
+    assertThat(cache.get(URI, c)).isNull();
+    verify(c).call();
+    assertCacheHit(false);
+  }
+
+  @Test
+  public void testClean() throws Exception {
+    assertCacheHit(false);
+    cache.clear();
+    assertCacheHit(false);
+  }
+
+  @Test
+  public void testCacheHit() throws Exception {
+    assertCacheHit(false);
+    assertCacheHit(true);
+  }
+
+  @Test
+  public void testForceUpdate() throws Exception {
+    cache = new PersistentCache(tmp.getRoot().toPath(), CACHE_EXPIRE, log, true);
+
+    assertCacheHit(false);
+    assertCacheHit(false);
+    assertCacheHit(false);
+
+    // with forceUpdate, it should still have cached the last call
+    cache = new PersistentCache(tmp.getRoot().toPath(), CACHE_EXPIRE, log, false);
+    assertCacheHit(true);
+  }
+
+  @Test
+  public void testExpiration() throws Exception {
+    assertCacheHit(false);
+    Thread.sleep(CACHE_EXPIRE);
+    assertCacheHit(false);
+  }
+
+  private void assertCacheHit(boolean hit) throws Exception {
+    CacheFillerString c = new CacheFillerString();
+    assertThat(cache.getString(URI, c)).isEqualTo(VALUE);
+    assertThat(c.wasCalled).isEqualTo(!hit);
+  }
+
+  private class CacheFillerString implements Callable<String> {
+    public boolean wasCalled = false;
+
+    @Override
+    public String call() throws Exception {
+      wasCalled = true;
+      return VALUE;
+    }
+  }
+
+  /**
+   * WSCache should be transparent regarding exceptions: if an exception is thrown by the value loader, it should pass through
+   * the cache to the original caller using the cache.
+   * @throws Exception 
+   */
+  @Test(expected = ArithmeticException.class)
+  public void testExceptions() throws Exception {
+    Callable<byte[]> c = mock(Callable.class);
+    when(c.call()).thenThrow(ArithmeticException.class);
+    cache.get(URI, c);
+  }
+
+}