SONAR-5329 - fixed IndexQueue compression loss due to SystemCurrentTime

diff --git a/sonar-server/src/main/java/org/sonar/server/log/index/LogIndex.java b/sonar-server/src/main/java/org/sonar/server/log/index/LogIndex.java
index bd5f6b2c7174984e561b37df47c366e908e6aae7..9e677ee94aa096064bfcb69491a0ea965c0c4c43 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/log/index/LogIndex.java
+++ b/sonar-server/src/main/java/org/sonar/server/log/index/LogIndex.java
@@ -87,6 +87,7 @@ public class LogIndex extends BaseIndex<Log, LogDto, LogKey> {
     return new Result<Log>(this, getClient().prepareSearch(this.getIndexName())
       .setQuery(QueryBuilders.matchAllQuery())
       .setTypes(this.getIndexType())
+      .setSize(Integer.MAX_VALUE)
       .get());
   }
 

diff --git a/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java b/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java
index c85e8a9d400824fdd3a8e75d9e88c67f22e2b99b..8a1d4c016899f8e544295c923011a3b96e6ef41d 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java
@@ -19,11 +19,8 @@
  */
 package org.sonar.server.search;
 
-import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.LinkedListMultimap;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
@@ -32,13 +29,9 @@ import org.sonar.api.ServerComponent;
 import org.sonar.core.cluster.WorkQueue;
 import org.sonar.server.search.action.EmbeddedIndexAction;
 import org.sonar.server.search.action.IndexAction;
-import org.sonar.server.search.action.KeyIndexAction;
 
-import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -95,36 +88,26 @@ public class IndexQueue extends LinkedBlockingQueue<Runnable>
     } else if (actions.size() > 1) {
 
       /* Purge actions that would be overridden  */
-      Map<String, Integer> itemOffset = new HashMap<String, Integer>();
-      ArrayListMultimap<String, IndexAction> itemActions = ArrayListMultimap.create();
-      List<IndexAction> embeddedActions = new LinkedList<IndexAction>();
+      Long purgeStart = System.currentTimeMillis();
+      List<IndexAction> itemActions = Lists.newArrayList();
+      List<IndexAction> embeddedActions = Lists.newArrayList();
+
       for (IndexAction action : actions) {
         if(action.getClass().isAssignableFrom(EmbeddedIndexAction.class)){
           embeddedActions.add(action);
         } else {
-          String actionKey = action.getKey();
-          Integer offset = 0;
-          if(!itemOffset.containsKey(actionKey)){
-            itemOffset.put(actionKey, offset);
-            itemActions.put(actionKey, action);
-          } else {
-            offset = itemOffset.get(actionKey);
-            if(action.getClass().isAssignableFrom(KeyIndexAction.class)){
-              itemOffset.put(actionKey, 0);
-              itemActions.get(actionKey).set(0, action);
-            } else {
-              itemActions.get(actionKey).set(offset, action);
-            }
-          }
+          itemActions.add(action);
         }
       }
 
+      LOGGER.debug("INDEX - compressed {} items into {} in {}ms,",
+        actions.size(), itemActions.size() + embeddedActions.size(), System.currentTimeMillis() - purgeStart);
+
       try {
         /* execute all item actions */
-        Multimap<String, IndexAction> itemBulks = makeBulkByType(itemActions);
-          CountDownLatch itemLatch = new CountDownLatch(itemBulks.size());
+        CountDownLatch itemLatch = new CountDownLatch(itemActions.size());
         indexTime = System.currentTimeMillis();
-        for (IndexAction action : itemBulks.values()) {
+        for (IndexAction action : itemActions) {
           action.setLatch(itemLatch);
           this.offer(action, 1000, TimeUnit.SECONDS);
           types.add(action.getPayloadClass().getSimpleName());
@@ -167,13 +150,4 @@ public class IndexQueue extends LinkedBlockingQueue<Runnable>
         bcount, ecount, StringUtils.join(refreshes, ","));
     }
   }
-
-  private Multimap<String, IndexAction> makeBulkByType(ArrayListMultimap<String, IndexAction> actions) {
-    Multimap<String, IndexAction> bulks = LinkedListMultimap.create();
-    for (IndexAction action : actions.values()) {
-      bulks.put(action.getIndexType(), action);
-    }
-    return bulks;
-  }
-
 }

diff --git a/sonar-server/src/test/java/org/sonar/server/log/LogBackendMediumTest.java b/sonar-server/src/test/java/org/sonar/server/log/LogBackendMediumTest.java
index 055d5ef65efb415a0138d2556323e0e5039e17d8..2429498b3f2e33b97cb67c04ddf3cb9be842d147 100644 (file)
--- a/sonar-server/src/test/java/org/sonar/server/log/LogBackendMediumTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/log/LogBackendMediumTest.java
@@ -32,8 +32,11 @@ import org.sonar.core.persistence.DbSession;
 import org.sonar.server.db.DbClient;
 import org.sonar.server.log.db.LogDao;
 import org.sonar.server.log.index.LogIndex;
+import org.sonar.server.log.index.LogQuery;
+import org.sonar.server.search.QueryOptions;
 import org.sonar.server.tester.ServerTester;
 
+import java.util.Iterator;
 import java.util.Map;
 
 import static org.fest.assertions.Assertions.assertThat;
@@ -101,13 +104,28 @@ public class LogBackendMediumTest {
 
   @Test
   public void massive_insert() {
+
+    // 0 Assert no logs in DB
     assertThat(dao.findAll(dbSession)).hasSize(0);
-    int max = 20;
+    int max = 200;
     final String testValue = "hello world";
     for (int i = 0; i < max; i++) {
       service.write(dbSession, Log.Type.ACTIVE_RULE, testValue + "_" + i);
     }
     dbSession.commit();
+
+    // 1. assert both backends have all logs
     assertThat(dao.findAll(dbSession)).hasSize(max);
+    assertThat(index.findAll().getHits()).hasSize(max);
+
+    // 2. assert scrollable
+    int count = 0;
+    Iterator<Log> logs = index.search(new LogQuery(), new QueryOptions().setScroll(true)).scroll();
+    while (logs.hasNext()) {
+      logs.next();
+      count++;
+    }
+    assertThat(count).isEqualTo(max);
+
   }
 }