SONAR-4360 OutOfMemory when creating thousands of issues

pom.xml

@@ -991,6 +991,11 @@
         <artifactId>jetty-servlets</artifactId>
         <version>${jetty.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.skyscreamer</groupId>
+        <artifactId>jsonassert</artifactId>
+        <version>1.2.0</version>
+      </dependency>
       <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>

sonar-batch/pom.xml

@@ -93,5 +93,10 @@
       <artifactId>jetty-server</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.skyscreamer</groupId>
+      <artifactId>jsonassert</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

sonar-batch/src/main/java/org/sonar/batch/scan/JsonReport.java

@@ -22,8 +22,7 @@ package org.sonar.batch.scan;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.Closeables;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
+import com.google.gson.stream.JsonWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.BatchComponent;
@@ -39,8 +38,6 @@ import org.sonar.core.i18n.RuleI18nManager;
 import org.sonar.core.issue.DefaultIssue;
 
 import java.io.*;
-import java.util.Collection;
-import java.util.Date;
 import java.util.Locale;
 import java.util.Set;
 
@@ -80,7 +77,8 @@ public class JsonReport implements BatchComponent {
     Writer output = null;
     try {
       output = new BufferedWriter(new FileWriter(exportFile));
-      createJson().writeJSONString(output);
+      writeJson(output);
+
     } catch (IOException e) {
       throw new SonarException("Unable to write report results in file " + exportFile.getAbsolutePath(), e);
     } finally {
@@ -89,85 +87,84 @@ public class JsonReport implements BatchComponent {
   }
 
   @VisibleForTesting
-  JSONObject createJson(){
-    Set<RuleKey> ruleKeyList = newHashSet();
-    Set<String> componentKeyList = newHashSet();
-
-    JSONObject json = new JSONObject();
-    put(json, "version", server.getVersion());
-
-    addIssues(json, ruleKeyList, componentKeyList);
-    addComponents(json, componentKeyList);
-    addRules(json, ruleKeyList);
-    return json;
-  }
-
-  private void addIssues(JSONObject root, Collection<RuleKey> ruleKeyList, Collection<String> componentKeyList) {
-    JSONArray json = new JSONArray();
-    for (DefaultIssue issue : getIssues()) {
-      JSONObject jsonIssue = new JSONObject();
-      put(jsonIssue, "key", issue.key());
-      put(jsonIssue, "component", issue.componentKey());
-      put(jsonIssue, "line", issue.line());
-      put(jsonIssue, "message", issue.message());
-      put(jsonIssue, "severity", issue.severity());
-      put(jsonIssue, "rule", issue.ruleKey());
-      put(jsonIssue, "status", issue.status());
-      put(jsonIssue, "resolution", issue.resolution());
-      put(jsonIssue, "isNew", issue.isNew());
-      put(jsonIssue, "reporter", issue.reporter());
-      put(jsonIssue, "assignee", issue.assignee());
-      put(jsonIssue, "effortToFix", issue.effortToFix());
-      put(jsonIssue, "creationDate", issue.creationDate());
-      put(jsonIssue, "updateDate", issue.updateDate());
-      put(jsonIssue, "closeDate", issue.closeDate());
-      json.add(jsonIssue);
-
-      componentKeyList.add(issue.componentKey());
-      ruleKeyList.add(issue.ruleKey());
-    }
-    root.put("issues", json);
-  }
-
-  private void addComponents(JSONObject root, Collection<String> componentKeyList) {
-    JSONArray json = new JSONArray();
-    for (String componentKey : componentKeyList) {
-      JSONObject jsonComponent = new JSONObject();
-      put(jsonComponent, "key", componentKey);
-      json.add(jsonComponent);
-    }
-    root.put("components", json);
-  }
+  void writeJson(Writer writer) {
+    JsonWriter json = null;
+    try {
+      json = new JsonWriter(writer);
+      json.setSerializeNulls(false);
+      json.beginObject();
+      json.name("version").value(server.getVersion());
+
+      Set<RuleKey> ruleKeys = newHashSet();
+      Set<String> componentKeys = newHashSet();
+      writeJsonIssues(json, ruleKeys, componentKeys);
+      writeJsonComponents(json, componentKeys);
+      writeJsonRules(json, ruleKeys);
+      json.endObject().flush();
 
-  private void addRules(JSONObject root, Collection<RuleKey> ruleKeyList) {
-    JSONArray json = new JSONArray();
-    for (RuleKey ruleKey : ruleKeyList) {
-      JSONObject jsonRuleKey = new JSONObject();
-      put(jsonRuleKey, "key", ruleKey);
-      put(jsonRuleKey, "rule", ruleKey.rule());
-      put(jsonRuleKey, "repository", ruleKey.repository());
-      put(jsonRuleKey, "name", getRuleName(ruleKey));
-      json.add(jsonRuleKey);
+    } catch (IOException e) {
+      throw new SonarException("Unable to write JSON report", e);
+    } finally {
+      Closeables.closeQuietly(json);
     }
-    root.put("rules", json);
   }
 
-  private void put(JSONObject json, String key, Object value) {
-    if (value != null) {
-      json.put(key, value);
+  private void writeJsonIssues(JsonWriter json, Set<RuleKey> ruleKeys, Set<String> componentKeys) throws IOException {
+    json.name("issues").beginArray();
+    for (DefaultIssue issue : getIssues()) {
+      json
+        .beginObject()
+        .name("key").value(issue.key())
+        .name("component").value(issue.componentKey())
+        .name("line").value(issue.line())
+        .name("message").value(issue.message())
+        .name("severity").value(issue.severity())
+        .name("rule").value(issue.ruleKey().toString())
+        .name("status").value(issue.status())
+        .name("resolution").value(issue.resolution())
+        .name("isNew").value(issue.isNew())
+        .name("reporter").value(issue.reporter())
+        .name("assignee").value(issue.assignee())
+        .name("effortToFix").value(issue.effortToFix());
+      if (issue.creationDate() != null) {
+        json.name("creationDate").value(DateUtils.formatDateTime(issue.creationDate()));
+      }
+      if (issue.updateDate() != null) {
+        json.name("updateDate").value(DateUtils.formatDateTime(issue.updateDate()));
+      }
+      if (issue.closeDate() != null) {
+        json.name("closeDate").value(DateUtils.formatDateTime(issue.closeDate()));
+      }
+      json.endObject();
+      componentKeys.add(issue.componentKey());
+      ruleKeys.add(issue.ruleKey());
     }
+    json.endArray();
   }
 
-  private void put(JSONObject json, String key, RuleKey ruleKey) {
-    if (ruleKey != null) {
-      json.put(key, ruleKey.toString());
+  private void writeJsonComponents(JsonWriter json, Set<String> componentKeys) throws IOException {
+    json.name("components").beginArray();
+    for (String componentKey : componentKeys) {
+      json
+        .beginObject()
+        .name("key").value(componentKey)
+        .endObject();
     }
+    json.endArray();
   }
 
-  private void put(JSONObject json, String key, Date date) {
-    if (date != null) {
-      json.put(key, DateUtils.formatDateTime(date));
+  private void writeJsonRules(JsonWriter json, Set<RuleKey> ruleKeys) throws IOException {
+    json.name("rules").beginArray();
+    for (RuleKey ruleKey : ruleKeys) {
+      json
+        .beginObject()
+        .name("key").value(ruleKey.toString())
+        .name("rule").value(ruleKey.rule())
+        .name("repository").value(ruleKey.repository())
+        .name("name").value(getRuleName(ruleKey))
+        .endObject();
     }
+    json.endArray();
   }
 
   private String getRuleName(RuleKey ruleKey) {

sonar-batch/src/test/java/org/sonar/batch/scan/JsonReportTest.java

@@ -21,11 +21,11 @@
 package org.sonar.batch.scan;
 
 import com.google.common.collect.Lists;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
+import org.json.JSONException;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.skyscreamer.jsonassert.JSONAssert;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.config.Settings;
 import org.sonar.api.issue.Issue;
@@ -36,12 +36,13 @@ import org.sonar.api.rules.Rule;
 import org.sonar.api.scan.filesystem.ModuleFileSystem;
 import org.sonar.api.utils.DateUtils;
 import org.sonar.batch.issue.IssueCache;
-import org.sonar.batch.scan.JsonReport;
 import org.sonar.core.i18n.RuleI18nManager;
 import org.sonar.core.issue.DefaultIssue;
+import org.sonar.test.TestUtils;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.StringWriter;
 import java.util.Collections;
 import java.util.Locale;
 
@@ -63,7 +64,7 @@ public class JsonReportTest {
   IssueCache issueCache = mock(IssueCache.class);
 
   @Before
-  public void before() {
+  public void setUp() {
     when(resource.getEffectiveKey()).thenReturn("Action.java");
     when(server.getVersion()).thenReturn("3.6");
 
@@ -73,59 +74,11 @@ public class JsonReportTest {
   }
 
   @Test
-  public void should_export_json() {
+  public void should_write_json() throws JSONException {
     DefaultIssue issue = new DefaultIssue()
       .setKey("200")
-      .setComponentKey("Action.java")
-      .setRuleKey(RuleKey.of("squid", "AvoidCycle"))
-      .setNew(false);
-
-    when(jsonReport.getIssues()).thenReturn(Lists.<DefaultIssue>newArrayList(issue));
-
-    JSONObject json = jsonReport.createJson();
-    assertThat(json.values()).hasSize(4);
-
-    assertThat(json.get("version")).isEqualTo("3.6");
-
-    assertThat(json.get("components")).isNotNull();
-    JSONArray components = (JSONArray) json.get("components");
-    assertThat(components).hasSize(1);
-
-    assertThat(json.get("issues")).isNotNull();
-    JSONArray issues = (JSONArray) json.get("issues");
-    assertThat(issues).hasSize(1);
-
-    assertThat(json.get("rules")).isNotNull();
-    JSONArray rules = (JSONArray) json.get("rules");
-    assertThat(rules).hasSize(1);
-  }
-
-  @Test
-  public void should_export_components() {
-    DefaultIssue issue = new DefaultIssue()
-      .setKey("200")
-      .setComponentKey("Action.java")
-      .setRuleKey(RuleKey.of("squid", "AvoidCycle"))
-      .setNew(false);
-
-    when(jsonReport.getIssues()).thenReturn(Lists.<DefaultIssue>newArrayList(issue));
-
-    JSONObject json = jsonReport.createJson();
-    assertThat(json.get("version")).isEqualTo("3.6");
-
-    assertThat(json.get("components")).isNotNull();
-    JSONArray components = (JSONArray) json.get("components");
-    assertThat(components).hasSize(1);
-    JSONObject jsonComponent = (JSONObject) components.get(0);
-    assertThat(jsonComponent.values()).hasSize(1);
-    assertThat(jsonComponent.get("key")).isEqualTo("Action.java");
-  }
-
-  @Test
-  public void should_export_issues() {
-    DefaultIssue issue = new DefaultIssue()
-      .setKey("200")
-      .setComponentKey("Action.java")
+      .setComponentKey("struts:org.apache.struts.Action")
+      .setRuleKey(RuleKey.of("squid", "AvoidCycles"))
       .setMessage("SystemPrintln")
       .setSeverity("MINOR")
       .setStatus(Issue.STATUS_CLOSED)
@@ -134,103 +87,37 @@ public class JsonReportTest {
       .setEffortToFix(3.14)
       .setReporter("julien")
       .setAssignee("simon")
-      .setRuleKey(RuleKey.of("squid", "AvoidCycle"))
+      .setRuleKey(RuleKey.of("squid", "AvoidCycles"))
       .setCreationDate(DateUtils.parseDate("2013-04-24"))
       .setUpdateDate(DateUtils.parseDate("2013-04-25"))
       .setCloseDate(DateUtils.parseDate("2013-04-26"))
       .setNew(false);
-
+    when(ruleI18nManager.getName("squid", "AvoidCycles", Locale.getDefault())).thenReturn("Avoid Cycles");
     when(jsonReport.getIssues()).thenReturn(Lists.<DefaultIssue>newArrayList(issue));
 
-    JSONObject json = jsonReport.createJson();
-    assertThat(json.get("issues")).isNotNull();
-    JSONArray issues = (JSONArray) json.get("issues");
-    assertThat(issues).hasSize(1);
-    JSONObject jsonIssue = (JSONObject) issues.get(0);
-    assertThat(jsonIssue.values()).hasSize(15);
-
-    assertThat(jsonIssue.get("key")).isEqualTo("200");
-    assertThat(jsonIssue.get("component")).isEqualTo("Action.java");
-    assertThat(jsonIssue.get("line")).isEqualTo(1);
-    assertThat(jsonIssue.get("message")).isEqualTo("SystemPrintln");
-    assertThat(jsonIssue.get("severity")).isEqualTo("MINOR");
-    assertThat(jsonIssue.get("rule")).isEqualTo("squid:AvoidCycle");
-    assertThat(jsonIssue.get("status")).isEqualTo("CLOSED");
-    assertThat(jsonIssue.get("resolution")).isEqualTo("FALSE-POSITIVE");
-    assertThat(jsonIssue.get("assignee")).isEqualTo("simon");
-    assertThat(jsonIssue.get("effortToFix")).isEqualTo(3.14);
-    assertThat(jsonIssue.get("reporter")).isEqualTo("julien");
-    assertThat(jsonIssue.get("isNew")).isEqualTo(false);
-    assertThat((String) jsonIssue.get("creationDate")).contains("2013-04-24T00:00");
-    assertThat((String) jsonIssue.get("updateDate")).contains("2013-04-25T00:00");
-    assertThat((String) jsonIssue.get("closeDate")).contains("2013-04-26T00:00");
-  }
-
-  @Test
-  public void should_export_rules() {
-    DefaultIssue issue = new DefaultIssue()
-      .setKey("200")
-      .setComponentKey("Action.java")
-      .setRuleKey(RuleKey.of("squid", "AvoidCycle"));
-
-    when(ruleI18nManager.getName("squid", "AvoidCycle", Locale.getDefault())).thenReturn("Avoid Cycle");
-    when(jsonReport.getIssues()).thenReturn(Lists.<DefaultIssue>newArrayList(issue));
-
-    JSONObject root = jsonReport.createJson();
-
-    assertThat(root.get("rules")).isNotNull();
-    JSONArray rules = (JSONArray) root.get("rules");
-    assertThat(rules).hasSize(1);
-    JSONObject json = (JSONObject) rules.get(0);
-    assertThat(json.values()).hasSize(4);
+    StringWriter writer = new StringWriter();
+    jsonReport.writeJson(writer);
 
-    assertThat(json.get("key")).isEqualTo("squid:AvoidCycle");
-    assertThat(json.get("rule")).isEqualTo("AvoidCycle");
-    assertThat(json.get("repository")).isEqualTo("squid");
-    assertThat(json.get("name")).isEqualTo("Avoid Cycle");
+    JSONAssert.assertEquals(TestUtils.getResourceContent("/org/sonar/batch/scan/JsonReportTest/report.json"),
+      writer.toString(), false);
   }
 
   @Test
-  public void should_export_issues_with_no_line() {
-    DefaultIssue issue = new DefaultIssue()
-      .setKey("200")
-      .setComponentKey("Action.java")
-      .setLine(null)
-      .setRuleKey(RuleKey.of("squid", "AvoidCycle"));
-
-    when(jsonReport.getIssues()).thenReturn(Lists.<DefaultIssue>newArrayList(issue));
-
-    JSONObject json = jsonReport.createJson();
-    assertThat(json.get("issues")).isNotNull();
-
-    JSONArray issues = (JSONArray) json.get("issues");
-    JSONObject jsonIssue = (JSONObject) issues.get(0);
-    assertThat(jsonIssue.get("key")).isEqualTo("200");
-    assertThat(jsonIssue.containsKey("line")).isFalse();
-  }
-
-  @Test
-  public void should_ignore_resources_without_issue() {
+  public void should_ignore_components_without_issue() throws JSONException {
     when(jsonReport.getIssues()).thenReturn(Collections.<DefaultIssue>emptyList());
 
-    JSONObject json = jsonReport.createJson();
-    assertThat(json.get("version")).isEqualTo("3.6");
-
-    assertThat(json.get("components")).isNotNull();
-    JSONArray components = (JSONArray) json.get("components");
-    assertThat(components).isEmpty();
+    StringWriter writer = new StringWriter();
+    jsonReport.writeJson(writer);
 
-    assertThat(json.get("issues")).isNotNull();
-    JSONArray issues = (JSONArray) json.get("issues");
-    assertThat(issues).isEmpty();
+    JSONAssert.assertEquals("{\"version\":\"3.6\"}", writer.toString(), false);
   }
 
   @Test
   public void should_export_issues_to_file() throws IOException {
     File sonarDirectory = temporaryFolder.newFolder("sonar");
 
-    Rule rule = Rule.create("squid", "AvoidCycle");
-    when(ruleI18nManager.getName(rule, Locale.getDefault())).thenReturn("Avoid Cycle");
+    Rule rule = Rule.create("squid", "AvoidCycles");
+    when(ruleI18nManager.getName(rule, Locale.getDefault())).thenReturn("Avoid Cycles");
     when(jsonReport.getIssues()).thenReturn(Collections.<DefaultIssue>emptyList());
 
     settings.setProperty("sonar.report.export.path", "output.json");

sonar-batch/src/test/resources/org/sonar/batch/scan/JsonReportTest/report.json

@@ -0,0 +1,29 @@
+{"version": "3.6", "issues": [
+  {
+    "key": "200",
+    "component": "struts:org.apache.struts.Action",
+    "line": 1,
+    "message": "SystemPrintln",
+    "severity": "MINOR",
+    "rule": "squid:AvoidCycles",
+    "status": "CLOSED",
+    "resolution": "FALSE-POSITIVE",
+    "isNew": false,
+    "reporter": "julien",
+    "assignee": "simon",
+    "effortToFix": 3.14,
+    "creationDate": "2013-04-24T00:00:00+0200",
+    "updateDate": "2013-04-25T00:00:00+0200",
+    "closeDate": "2013-04-26T00:00:00+0200"
+  }
+], "components": [
+  {
+    "key": "struts:org.apache.struts.Action"
+  }
+], "rules": [
+  {
+    "key": "squid:AvoidCycles",
+    "rule": "AvoidCycles",
+    "repository": "squid"
+  }
+]}