SONAR-6956 Index issues by project

When computing a report of a project, instead of indexing all changed issues, only updated issues from the project are now indexed.

diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexIssuesStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexIssuesStep.java
index 860eb1dc544de98dc878a98f74cb87f709e5d1eb..87e94cbc1cec418c857a097a2eaa1e0b83d6cc90 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexIssuesStep.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexIssuesStep.java
@@ -20,19 +20,22 @@
 
 package org.sonar.server.computation.step;
 
+import org.sonar.server.computation.component.TreeRootHolder;
 import org.sonar.server.issue.index.IssueIndexer;
 
 public class IndexIssuesStep implements ComputationStep {
 
   private final IssueIndexer indexer;
+  private final TreeRootHolder treeRootHolder;
 
-  public IndexIssuesStep(IssueIndexer indexer) {
+  public IndexIssuesStep(IssueIndexer indexer, TreeRootHolder treeRootHolder) {
     this.indexer = indexer;
+    this.treeRootHolder = treeRootHolder;
   }
 
   @Override
   public void execute() {
-    indexer.index();
+    indexer.index(treeRootHolder.getRoot().getUuid());
   }
 
   @Override

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
index d7ff6d85eb0b338bca610a3977ae0619bf740b14..688297c425a518e2f0645661d04976843009be75 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
@@ -20,6 +20,7 @@
 package org.sonar.server.issue.index;
 
 import java.util.Iterator;
+import javax.annotation.Nullable;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.index.query.FilterBuilders;
@@ -41,11 +42,20 @@ public class IssueIndexer extends BaseIndexer {
 
   @Override
   protected long doIndex(long lastUpdatedAt) {
-    return doIndex(createBulkIndexer(false), lastUpdatedAt);
+    return doIndex(createBulkIndexer(false), lastUpdatedAt, null);
   }
 
   public void indexAll() {
-    doIndex(createBulkIndexer(true), 0L);
+    doIndex(createBulkIndexer(true), 0L, null);
+  }
+
+  public void index(final String projectUuid) {
+    super.index(new IndexerTask() {
+      @Override
+      public long index(long lastUpdatedAt) {
+        return doIndex(createBulkIndexer(false), lastUpdatedAt, projectUuid);
+      }
+    });
   }
 
   /**
@@ -55,11 +65,11 @@ public class IssueIndexer extends BaseIndexer {
     doIndex(createBulkIndexer(false), issues);
   }
 
-  private long doIndex(BulkIndexer bulk, long lastUpdatedAt) {
+  private long doIndex(BulkIndexer bulk, long lastUpdatedAt, @Nullable String projectUuid) {
     DbSession dbSession = dbClient.openSession(false);
     long maxDate;
     try {
-      IssueResultSetIterator rowIt = IssueResultSetIterator.create(dbClient, dbSession, lastUpdatedAt);
+      IssueResultSetIterator rowIt = IssueResultSetIterator.create(dbClient, dbSession, lastUpdatedAt, projectUuid);
       maxDate = doIndex(bulk, rowIt);
       rowIt.close();
       return maxDate;

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java
index e60c4b65a32c9b9a1c07c306a7590b915bdc5702..dfc4ea6ac77aee75875b78a86fc3e916e3fad82f 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java
@@ -90,6 +90,8 @@ class IssueResultSetIterator extends ResultSetIterator<IssueDoc> {
 
   private static final String SQL_AFTER_DATE = SQL_ALL + " where i.updated_at>?";
 
+  private static final String PROJECT_FILTER = " AND root.uuid=?";
+
   private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
 
   private static final Splitter MODULE_PATH_SPLITTER = Splitter.on('.').trimResults().omitEmptyStrings();
@@ -98,12 +100,18 @@ class IssueResultSetIterator extends ResultSetIterator<IssueDoc> {
     super(stmt);
   }
 
-  static IssueResultSetIterator create(DbClient dbClient, DbSession session, long afterDate) {
+  static IssueResultSetIterator create(DbClient dbClient, DbSession session, long afterDate, @Nullable String projectUuid) {
     try {
       String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL;
+      sql += projectUuid == null ? "" : PROJECT_FILTER;
       PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
+      int index = 1;
       if (afterDate > 0L) {
-        stmt.setLong(1, afterDate);
+        stmt.setLong(index, afterDate);
+        index++;
+      }
+      if (projectUuid != null) {
+        stmt.setString(index, projectUuid);
       }
       return new IssueResultSetIterator(stmt);
     } catch (SQLException e) {

diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexIssuesStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexIssuesStepTest.java
index 06e7456b9bf9aa12e01bd0bafd007ee40c683103..22ac883db0c95d2a760c1a241654c57d3fd0ead9 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexIssuesStepTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexIssuesStepTest.java
@@ -20,21 +20,33 @@
 
 package org.sonar.server.computation.step;
 
+import org.junit.Rule;
 import org.junit.Test;
+import org.sonar.server.computation.batch.TreeRootHolderRule;
+import org.sonar.server.computation.component.Component;
+import org.sonar.server.computation.component.ReportComponent;
 import org.sonar.server.issue.index.IssueIndexer;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
+import static org.sonar.server.computation.component.Component.Type.*;
+import static org.sonar.server.computation.component.ReportComponent.*;
 
 public class IndexIssuesStepTest {
 
+  static String PROJECT_UUID = "PROJECT_UUID";
+
+  @Rule
+  public TreeRootHolderRule treeRootHolder = new TreeRootHolderRule()
+    .setRoot(builder(PROJECT, 1).setUuid(PROJECT_UUID).setKey("PROJECT_KEY").build());
+
   @Test
   public void call_indexers() {
     IssueIndexer issueIndexer = mock(IssueIndexer.class);
-    IndexIssuesStep underTest = new IndexIssuesStep(issueIndexer);
+    IndexIssuesStep underTest = new IndexIssuesStep(issueIndexer, treeRootHolder);
 
     underTest.execute();
 
-    verify(issueIndexer).index();
+    verify(issueIndexer).index(PROJECT_UUID);
   }
 }

diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
index 36f5853acbf28beafbe6ad1b25386f98ae8273a3..a5504688b735c29f453a55885b684493144126d6 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
@@ -39,11 +39,10 @@ import static org.assertj.core.api.Assertions.assertThat;
 @Category(DbTests.class)
 public class IssueIndexerTest {
 
-  @Rule
-  public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
   @ClassRule
   public static EsTester esTester = new EsTester().addDefinitions(new IssueIndexDefinition(new Settings()));
+  @Rule
+  public DbTester dbTester = DbTester.create(System2.INSTANCE);
 
   @Before
   public void setUp() {
@@ -77,6 +76,7 @@ public class IssueIndexerTest {
     List<IssueDoc> docs = esTester.getDocuments("issues", "issue", IssueDoc.class);
     assertThat(docs).hasSize(1);
     IssueDoc doc = docs.get(0);
+    assertThat(doc.key()).isEqualTo("ABCDE");
     assertThat(doc.projectUuid()).isEqualTo("THE_PROJECT");
     assertThat(doc.componentUuid()).isEqualTo("THE_FILE");
     assertThat(doc.moduleUuid()).isEqualTo("THE_PROJECT");
@@ -90,13 +90,34 @@ public class IssueIndexerTest {
 
     // technical date
     assertThat(doc.getTechnicalUpdateDate().getTime()).isEqualTo(1550000000000L);
+  }
+
+  @Test
+  public void delete_project_remove_issue() {
+    dbTester.prepareDbUnit(getClass(), "index.xml");
+
+    IssueIndexer indexer = createIndexer();
+    indexer.index();
+
+    assertThat(esTester.countDocuments("issues", "issue")).isEqualTo(1);
 
-    // delete project
     indexer.deleteProject("THE_PROJECT", true);
 
     assertThat(esTester.countDocuments("issues", "issue")).isZero();
   }
 
+  @Test
+  public void index_issues_from_project() {
+    dbTester.prepareDbUnit(getClass(), "index_project.xml");
+
+    IssueIndexer indexer = createIndexer();
+    indexer.index("THE_PROJECT_1");
+
+    List<IssueDoc> docs = esTester.getDocuments("issues", "issue", IssueDoc.class);
+    assertThat(docs).hasSize(1);
+    assertThat(docs.get(0).key()).isEqualTo("ABCDE");
+  }
+
   private IssueIndexer createIndexer() {
     IssueIndexer indexer = new IssueIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client());
     indexer.setEnabled(true);

diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java
index 9ecc7685721f73f05fea0c1abf91f639b8ddeb23..8cfe8bc91548ee6d4e06a38ee99ef80e1b900b40 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java
@@ -22,6 +22,7 @@ package org.sonar.server.issue.index;
 import com.google.common.base.Function;
 import com.google.common.collect.Maps;
 import java.util.Map;
+import javax.annotation.Nonnull;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -39,6 +40,15 @@ public class IssueResultSetIteratorTest {
   @Rule
   public DbTester dbTester = DbTester.create(System2.INSTANCE);
 
+  private static Map<String, IssueDoc> issuesByKey(IssueResultSetIterator it) {
+    return Maps.uniqueIndex(it, new Function<IssueDoc, String>() {
+      @Override
+      public String apply(@Nonnull IssueDoc issue) {
+        return issue.key();
+      }
+    });
+  }
+
   @Before
   public void setUp() {
     dbTester.truncateTables();
@@ -47,7 +57,7 @@ public class IssueResultSetIteratorTest {
   @Test
   public void iterator_over_one_issue() {
     dbTester.prepareDbUnit(getClass(), "one_issue.xml");
-    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L);
+    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L, null);
     Map<String, IssueDoc> issuesByKey = issuesByKey(it);
     it.close();
 
@@ -81,7 +91,7 @@ public class IssueResultSetIteratorTest {
   @Test
   public void iterator_over_issues() {
     dbTester.prepareDbUnit(getClass(), "shared.xml");
-    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L);
+    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L, null);
     Map<String, IssueDoc> issuesByKey = issuesByKey(it);
     it.close();
 
@@ -136,10 +146,30 @@ public class IssueResultSetIteratorTest {
     assertThat(issue.debt().toMinutes()).isGreaterThan(0L);
   }
 
+  @Test
+  public void iterator_over_issue_from_project() {
+    dbTester.prepareDbUnit(getClass(), "many_projects.xml");
+    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L, "THE_PROJECT_1");
+    Map<String, IssueDoc> issuesByKey = issuesByKey(it);
+    it.close();
+
+    assertThat(issuesByKey).hasSize(2);
+  }
+
+  @Test
+  public void iterator_over_issue_from_project_and_date() {
+    dbTester.prepareDbUnit(getClass(), "many_projects.xml");
+    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 1_600_000_000_000L, "THE_PROJECT_1");
+    Map<String, IssueDoc> issuesByKey = issuesByKey(it);
+    it.close();
+
+    assertThat(issuesByKey).hasSize(1);
+  }
+
   @Test
   public void extract_directory_path() {
     dbTester.prepareDbUnit(getClass(), "extract_directory_path.xml");
-    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L);
+    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L, null);
     Map<String, IssueDoc> issuesByKey = issuesByKey(it);
     it.close();
 
@@ -161,7 +191,7 @@ public class IssueResultSetIteratorTest {
   @Test
   public void extract_file_path() {
     dbTester.prepareDbUnit(getClass(), "extract_file_path.xml");
-    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L);
+    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L, null);
     Map<String, IssueDoc> issuesByKey = issuesByKey(it);
     it.close();
 
@@ -183,7 +213,7 @@ public class IssueResultSetIteratorTest {
   @Test
   public void select_after_date() {
     dbTester.prepareDbUnit(getClass(), "shared.xml");
-    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 1_420_000_000_000L);
+    IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 1_420_000_000_000L, null);
 
     assertThat(it.hasNext()).isTrue();
     IssueDoc issue = it.next();
@@ -192,13 +222,4 @@ public class IssueResultSetIteratorTest {
     assertThat(it.hasNext()).isFalse();
     it.close();
   }
-
-  private static Map<String, IssueDoc> issuesByKey(IssueResultSetIterator it) {
-    return Maps.uniqueIndex(it, new Function<IssueDoc, String>() {
-      @Override
-      public String apply(IssueDoc issue) {
-        return issue.key();
-      }
-    });
-  }
 }

diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueIndexerTest/index_project.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueIndexerTest/index_project.xml
new file mode 100644 (file)
index 0000000..0f83fdf
--- /dev/null
+++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueIndexerTest/index_project.xml
@@ -0,0 +1,72 @@
+<dataset>
+  <rules id="1" tags="[null]" system_tags="[null]" name="Avoid Cycles" plugin_rule_key="AvoidCycles"
+         plugin_config_key="[null]" plugin_name="squid"/>
+
+  <!-- Project 1 -->
+  <projects id="10" scope="PRJ" qualifier="TRK" kee="the_project_1" name="TheProject1"
+            uuid="THE_PROJECT_1" module_uuid="[null]" module_uuid_path="." path="[null]"/>
+  <projects id="11" scope="FIL" qualifier="FIL" kee="the_file_1" name="TheFile1"
+            uuid="THE_FILE_1" module_uuid="THE_PROJECT_1" module_uuid_path=".THE_PROJECT_1."
+            path="src/main/java/TheFile.java"/>
+
+  <issues id="1"
+          kee="ABCDE"
+          resolution="FIXED"
+          status="RESOLVED"
+          severity="BLOCKER"
+          manual_severity="[false]"
+          assignee="winner"
+          author_login="[null]"
+          checksum="FFFFF"
+          effort_to_fix="[null]"
+          technical_debt="[null]"
+          message="[null]"
+          line="444"
+          component_uuid="THE_FILE_1"
+          project_uuid="THE_PROJECT_1"
+          rule_id="1"
+          reporter="[null]"
+          issue_attributes="JIRA=http://jira.com"
+          action_plan_key="[null]"
+          created_at="1500000000000"
+          updated_at="1550000000000"
+          issue_creation_date="1115848800000"
+          issue_update_date="1368828000000"
+          issue_close_date="[null]"
+          locations="[null]"
+  />
+
+  <!-- Project 2 -->
+  <projects id="100" scope="PRJ" qualifier="TRK" kee="the_project_2" name="TheProject2"
+            uuid="THE_PROJECT_2" module_uuid="[null]" module_uuid_path="." path="[null]"/>
+  <projects id="111" scope="FIL" qualifier="FIL" kee="the_file_2" name="TheFile2"
+            uuid="THE_FILE_2" module_uuid="THE_PROJECT_2" module_uuid_path=".THE_PROJECT_2."
+            path="src/main/java/TheFile.java"/>
+
+  <issues id="10"
+          kee="EDCBA"
+          resolution="FIXED"
+          status="RESOLVED"
+          severity="BLOCKER"
+          manual_severity="[false]"
+          assignee="winner"
+          author_login="[null]"
+          checksum="FFFFF"
+          effort_to_fix="[null]"
+          technical_debt="[null]"
+          message="[null]"
+          line="444"
+          component_uuid="THE_FILE_2"
+          project_uuid="THE_PROJECT_2"
+          rule_id="1"
+          reporter="[null]"
+          issue_attributes="JIRA=http://jira.com"
+          action_plan_key="[null]"
+          created_at="1500000000000"
+          updated_at="1550000000000"
+          issue_creation_date="1115848800000"
+          issue_update_date="1368828000000"
+          issue_close_date="[null]"
+          locations="[null]"
+  />
+</dataset>

diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueResultSetIteratorTest/many_projects.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueResultSetIteratorTest/many_projects.xml
new file mode 100644 (file)
index 0000000..15c8120
--- /dev/null
+++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueResultSetIteratorTest/many_projects.xml
@@ -0,0 +1,99 @@
+<dataset>
+  <rules id="1" tags="[null]" system_tags="[null]" name="Avoid Cycles" plugin_rule_key="AvoidCycles"
+         plugin_config_key="[null]" plugin_name="squid"/>
+
+  <!-- Project 1 -->
+  <projects id="10" scope="PRJ" qualifier="TRK" kee="the_project_1" name="TheProject1"
+            uuid="THE_PROJECT_1" module_uuid="[null]" module_uuid_path="." path="[null]"/>
+  <projects id="11" scope="FIL" qualifier="FIL" kee="the_file_1" name="TheFile1"
+            uuid="THE_FILE_1" module_uuid="THE_PROJECT_1" module_uuid_path=".THE_PROJECT_1."
+            path="src/main/java/TheFile.java"/>
+
+  <issues id="1"
+          kee="ABCDE"
+          resolution="FIXED"
+          status="RESOLVED"
+          severity="BLOCKER"
+          manual_severity="[false]"
+          assignee="winner"
+          author_login="[null]"
+          checksum="FFFFF"
+          effort_to_fix="[null]"
+          technical_debt="[null]"
+          message="[null]"
+          line="444"
+          component_uuid="THE_FILE_1"
+          project_uuid="THE_PROJECT_1"
+          rule_id="1"
+          reporter="[null]"
+          issue_attributes="JIRA=http://jira.com"
+          action_plan_key="[null]"
+          created_at="1500000000000"
+          updated_at="1550000000000"
+          issue_creation_date="1115848800000"
+          issue_update_date="1368828000000"
+          issue_close_date="[null]"
+          locations="[null]"
+  />
+
+  <issues id="2"
+          kee="BCDEF"
+          resolution="FIXED"
+          status="RESOLVED"
+          severity="BLOCKER"
+          manual_severity="[false]"
+          assignee="winner"
+          author_login="[null]"
+          checksum="FFFFF"
+          effort_to_fix="[null]"
+          technical_debt="[null]"
+          message="[null]"
+          line="444"
+          component_uuid="THE_FILE_1"
+          project_uuid="THE_PROJECT_1"
+          rule_id="1"
+          reporter="[null]"
+          issue_attributes="JIRA=http://jira.com"
+          action_plan_key="[null]"
+          created_at="2000000000000"
+          updated_at="2550000000000"
+          issue_creation_date="2115848800000"
+          issue_update_date="2368828000000"
+          issue_close_date="[null]"
+          locations="[null]"
+  />
+
+  <!-- Project 2 -->
+  <projects id="100" scope="PRJ" qualifier="TRK" kee="the_project_2" name="TheProject2"
+            uuid="THE_PROJECT_2" module_uuid="[null]" module_uuid_path="." path="[null]"/>
+  <projects id="111" scope="FIL" qualifier="FIL" kee="the_file_2" name="TheFile2"
+            uuid="THE_FILE_2" module_uuid="THE_PROJECT_2" module_uuid_path=".THE_PROJECT_2."
+            path="src/main/java/TheFile.java"/>
+
+  <issues id="10"
+          kee="EDCBA"
+          resolution="FIXED"
+          status="RESOLVED"
+          severity="BLOCKER"
+          manual_severity="[false]"
+          assignee="winner"
+          author_login="[null]"
+          checksum="FFFFF"
+          effort_to_fix="[null]"
+          technical_debt="[null]"
+          message="[null]"
+          line="444"
+          component_uuid="THE_FILE_2"
+          project_uuid="THE_PROJECT_2"
+          rule_id="1"
+          reporter="[null]"
+          issue_attributes="JIRA=http://jira.com"
+          action_plan_key="[null]"
+          created_at="1500000000000"
+          updated_at="1550000000000"
+          issue_creation_date="1115848800000"
+          issue_update_date="1368828000000"
+          issue_close_date="[null]"
+          locations="[null]"
+  />
+</dataset>