Automatic merge from branch-5.0

* origin/branch-5.0:
  SONAR-5967 Remove the limitation of 3000 lines
  SONAR-5967 Instead of not displaying anymore the source files containing more than 3'000 lines, only the highlighting mechanism should be deactivated above this limit
  SONAR-5817 Fix max number of lines to 500000
  SONAR-5971 The component viewer, in differential mode, might display some unexpected issues

11 files changed, 129 insertions, 33 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/SourceService.java b/server/sonar-server/src/main/java/org/sonar/server/source/SourceService.java
index 4b4add00ce5..b65ebd98050 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/SourceService.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/SourceService.java
@@ -77,6 +77,10 @@ public class SourceService implements ServerComponent {
     return lines;
   }
 
+  public long countLines(String fileUuid){
+    return sourceLineIndex.countLines(fileUuid);
+  }
+
   @CheckForNull
   public String getScmAuthorData(String fileKey) {
     checkPermission(fileKey);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
index 7269ceb8777..4567a0f11ba 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
@@ -31,6 +31,8 @@ import java.util.List;
 
 public class SourceLineIndex implements ServerComponent {
 
+  private static final int MAX_RESULT = 500000;
+
   private final EsClient esClient;
 
   public SourceLineIndex(EsClient esClient) {
@@ -41,6 +43,8 @@ public class SourceLineIndex implements ServerComponent {
    * Get lines of code for file with UUID <code>fileUuid</code> with line numbers
    * between <code>from</code> and <code>to</code> (both inclusive). Line numbers
    * start at 1.
+   * The max number of returned lines will be 500000.
+   *
    * @param fileUuid the UUID of the file for which to get source code
    * @param from starting line; must be strictly positive
    * @param to ending line; must be greater than or equal to <code>to</code>
@@ -49,15 +53,20 @@ public class SourceLineIndex implements ServerComponent {
     Preconditions.checkArgument(from > 0, "Minimum value for 'from' is 1");
     Preconditions.checkArgument(to >= from, "'to' must be larger than or equal to 'from'");
     List<SourceLineDoc> lines = Lists.newArrayList();
+    int size = 1 + to - from;
+    if (size > MAX_RESULT) {
+      size = MAX_RESULT;
+    }
+    int toLimited = size + from - 1;
 
     for (SearchHit hit: esClient.prepareSearch(SourceLineIndexDefinition.INDEX)
       .setTypes(SourceLineIndexDefinition.TYPE)
-      .setSize(1 + to - from)
+      .setSize(size)
       .setQuery(QueryBuilders.boolQuery()
         .must(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid))
         .must(QueryBuilders.rangeQuery(SourceLineIndexDefinition.FIELD_LINE)
           .gte(from)
-          .lte(to)))
+          .lte(toLimited)))
       .addSort(SourceLineIndexDefinition.FIELD_LINE, SortOrder.ASC)
       .get().getHits().getHits()) {
       lines.add(new SourceLineDoc(hit.sourceAsMap()));
@@ -65,4 +74,12 @@ public class SourceLineIndex implements ServerComponent {
 
     return lines;
   }
+
+  public long countLines(String fileUuid) {
+    return esClient.prepareCount(SourceLineIndexDefinition.INDEX)
+      .setTypes(SourceLineIndexDefinition.TYPE)
+      .setQuery(QueryBuilders.boolQuery()
+        .must(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid)))
+      .get().getCount();
+  }
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/ws/ShowAction.java b/server/sonar-server/src/main/java/org/sonar/server/source/ws/ShowAction.java
index 9fb60b42a2e..3ba39a90169 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/ws/ShowAction.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/ws/ShowAction.java
@@ -30,7 +30,6 @@ import org.sonar.api.web.UserRole;
 import org.sonar.core.component.ComponentDto;
 import org.sonar.core.persistence.DbSession;
 import org.sonar.server.db.DbClient;
-import org.sonar.server.exceptions.NotFoundException;
 import org.sonar.server.source.SourceService;
 import org.sonar.server.user.UserSession;
 
@@ -79,7 +78,6 @@ public class ShowAction implements RequestHandler {
   @Override
   public void handle(Request request, Response response) {
     String fileKey = request.mandatoryParam("key");
-    UserSession.get().checkComponentPermission(UserRole.CODEVIEWER, fileKey);
 
     int from = Math.max(request.mandatoryParamAsInt("from"), 1);
     int to = (Integer) ObjectUtils.defaultIfNull(request.paramAsInt("to"), Integer.MAX_VALUE);
@@ -87,11 +85,18 @@ public class ShowAction implements RequestHandler {
     DbSession session = dbClient.openSession(false);
     try {
       ComponentDto componentDto = dbClient.componentDao().getByKey(session, fileKey);
-      List<String> linesHtml = sourceService.getLinesAsHtml(componentDto.uuid(), from, to);
-      if (linesHtml == null) {
-        throw new NotFoundException("File '" + fileKey + "' does not exist");
-      }
+      UserSession.get().checkComponentPermission(UserRole.CODEVIEWER, fileKey);
+
+      long linesSize = sourceService.countLines(componentDto.uuid());
+      int size = to - from;
+      boolean disableHighlighting = size > 3000 && linesSize > 3000 ;
 
+      List<String> linesHtml;
+      if (!disableHighlighting) {
+        linesHtml = sourceService.getLinesAsHtml(componentDto.uuid(), from, to);
+      } else {
+        linesHtml = sourceService.getLinesAsTxt(componentDto.uuid(), from, to);
+      }
       JsonWriter json = response.newJsonWriter().beginObject();
       writeSource(linesHtml, from, json);
 
@@ -99,7 +104,6 @@ public class ShowAction implements RequestHandler {
     } finally {
       session.close();
     }
-
   }
 
   private void writeSource(List<String> lines, int from, JsonWriter json) {
diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/SourceServiceTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/SourceServiceTest.java
index aac22010b27..b7f5c2e58dc 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/source/SourceServiceTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/SourceServiceTest.java
@@ -43,10 +43,7 @@ import static org.fest.assertions.Assertions.assertThat;
 import static org.fest.assertions.Fail.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
 
 @RunWith(MockitoJUnitRunner.class)
 public class SourceServiceTest {
@@ -152,4 +149,9 @@ public class SourceServiceTest {
     assertThat(result).contains("line1", "line2");
   }
 
+  @Test
+  public void count_lines() throws Exception {
+    service.countLines(COMPONENT_UUID);
+    verify(sourceLineIndex).countLines(COMPONENT_UUID);
+  }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java
index ced283fc0bb..8f1873dba9a 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java
@@ -63,4 +63,17 @@ public class SourceLineIndexTest {
   public void should_reject_to_less_than_from() {
     index.getLines("polop", 2, 1);
   }
+
+  @Test
+  public void count_lines() throws Exception {
+    es.putDocuments(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE,
+      this.getClass(),
+      "file1_line1.json",
+      "file1_line2.json",
+      "file2_line1.json",
+      "file2_line2.json",
+      "file2_line3.json");
+    assertThat(index.countLines("file1")).isEqualTo(2);
+    assertThat(index.countLines("file2")).isEqualTo(3);
+  }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/ws/ShowActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/ws/ShowActionTest.java
index c3471efa0bb..666d41691c5 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/source/ws/ShowActionTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/ws/ShowActionTest.java
@@ -38,9 +38,7 @@ import org.sonar.server.ws.WsTester;
 import static com.google.common.collect.Lists.newArrayList;
 import static org.mockito.Matchers.anyInt;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
 
 @RunWith(MockitoJUnitRunner.class)
 public class ShowActionTest {
@@ -76,6 +74,7 @@ public class ShowActionTest {
     String fileKey = "src/Foo.java";
     MockUserSession.set().addComponentPermission(UserRole.CODEVIEWER, "polop", fileKey);
     when(componentDao.getByKey(session, fileKey)).thenReturn(file);
+    when(sourceService.countLines(file.uuid())).thenReturn(6L);
     when(sourceService.getLinesAsHtml(eq(file.uuid()), anyInt(), anyInt())).thenReturn(newArrayList(
       "/*",
       " * Header",
@@ -94,6 +93,7 @@ public class ShowActionTest {
     String fileKey = "src/Foo.java";
     MockUserSession.set().addComponentPermission(UserRole.CODEVIEWER, "polop", fileKey);
     when(componentDao.getByKey(session, fileKey)).thenReturn(file);
+    when(sourceService.countLines(file.uuid())).thenReturn(6L);
     when(sourceService.getLinesAsHtml(file.uuid(), 3, 5)).thenReturn(newArrayList(
       " */",
       "",
@@ -112,6 +112,7 @@ public class ShowActionTest {
     String fileKey = "src/Foo.java";
     MockUserSession.set().addComponentPermission(UserRole.CODEVIEWER, "polop", fileKey);
     when(componentDao.getByKey(session, fileKey)).thenReturn(file);
+    when(sourceService.countLines(file.uuid())).thenReturn(6L);
     when(sourceService.getLinesAsHtml(file.uuid(), 1, 5)).thenReturn(newArrayList(
       " */",
       "",
@@ -126,6 +127,72 @@ public class ShowActionTest {
     verify(sourceService).getLinesAsHtml(file.uuid(), 1, 5);
   }
 
+  @Test
+  public void disable_highlighting_when_lines_greater_than_3000_without_from_and_to_params() throws Exception {
+    String fileKey = "src/Foo.java";
+    MockUserSession.set().addComponentPermission(UserRole.CODEVIEWER, "polop", fileKey);
+    when(componentDao.getByKey(session, fileKey)).thenReturn(file);
+    when(sourceService.countLines(file.uuid())).thenReturn(5000L);
+    WsTester.TestRequest request = tester
+      .newGetRequest("api/sources", "show")
+      .setParam("key", fileKey);
+    request.execute();
+
+    verify(sourceService).getLinesAsTxt(eq(file.uuid()), anyInt(), anyInt());
+    verify(sourceService, never()).getLinesAsHtml(eq(file.uuid()), anyInt(), anyInt());
+  }
+
+  @Test
+  public void disable_highlighting_when_lines_greater_than_3000_with_from_and_to_params() throws Exception {
+    String fileKey = "src/Foo.java";
+    MockUserSession.set().addComponentPermission(UserRole.CODEVIEWER, "polop", fileKey);
+    when(componentDao.getByKey(session, fileKey)).thenReturn(file);
+    when(sourceService.countLines(file.uuid())).thenReturn(5000L);
+    WsTester.TestRequest request = tester
+      .newGetRequest("api/sources", "show")
+      .setParam("key", fileKey)
+      .setParam("from", "1000")
+      .setParam("to", "5000");
+    request.execute();
+
+    verify(sourceService).getLinesAsTxt(eq(file.uuid()), anyInt(), anyInt());
+    verify(sourceService, never()).getLinesAsHtml(eq(file.uuid()), anyInt(), anyInt());
+  }
+
+  @Test
+  public void not_disable_highlighting_when_lines_smaller_than_3000_but_to_minus_to_greater_than_3000() throws Exception {
+    String fileKey = "src/Foo.java";
+    MockUserSession.set().addComponentPermission(UserRole.CODEVIEWER, "polop", fileKey);
+    when(componentDao.getByKey(session, fileKey)).thenReturn(file);
+    when(sourceService.countLines(file.uuid())).thenReturn(1000L);
+    WsTester.TestRequest request = tester
+      .newGetRequest("api/sources", "show")
+      .setParam("key", fileKey)
+      .setParam("from", "1000")
+      .setParam("to", "5000");
+    request.execute();
+
+    verify(sourceService, never()).getLinesAsTxt(eq(file.uuid()), anyInt(), anyInt());
+    verify(sourceService).getLinesAsHtml(eq(file.uuid()), anyInt(), anyInt());
+  }
+
+  @Test
+  public void not_disable_highlighting_when_lines_greater_than_3000_but_to_minus_to_smaller_than_3000() throws Exception {
+    String fileKey = "src/Foo.java";
+    MockUserSession.set().addComponentPermission(UserRole.CODEVIEWER, "polop", fileKey);
+    when(componentDao.getByKey(session, fileKey)).thenReturn(file);
+    when(sourceService.countLines(file.uuid())).thenReturn(5000L);
+    WsTester.TestRequest request = tester
+      .newGetRequest("api/sources", "show")
+      .setParam("key", fileKey)
+      .setParam("from", "1")
+      .setParam("to", "10");
+    request.execute();
+
+    verify(sourceService, never()).getLinesAsTxt(eq(file.uuid()), anyInt(), anyInt());
+    verify(sourceService).getLinesAsHtml(eq(file.uuid()), anyInt(), anyInt());
+  }
+
   @Test(expected = ForbiddenException.class)
   public void require_code_viewer() throws Exception {
     String fileKey = "src/Foo.java";
diff --git a/server/sonar-web/src/main/coffee/component-viewer/main.coffee b/server/sonar-web/src/main/coffee/component-viewer/main.coffee
index ab9241fcbed..f19d698d48d 100644
--- a/server/sonar-web/src/main/coffee/component-viewer/main.coffee
+++ b/server/sonar-web/src/main/coffee/component-viewer/main.coffee
@@ -190,8 +190,7 @@ define [
           @periods.reset [{label: t('none')}]
           data.periods.forEach (p) =>
             d = moment p[2]
-            date = new Date d.get('year'), d.get('month'), d.get('date')
-            p = @periods.add key: p[0], label: p[1], sinceDate: date
+            p = @periods.add key: p[0], label: p[1], sinceDate: d.toDate()
 
 
     requestMeasures: (key, period = null) ->
diff --git a/server/sonar-web/src/main/coffee/component-viewer/mixins/main-issues.coffee b/server/sonar-web/src/main/coffee/component-viewer/mixins/main-issues.coffee
index a0bf0148157..d4976ffd101 100644
--- a/server/sonar-web/src/main/coffee/component-viewer/mixins/main-issues.coffee
+++ b/server/sonar-web/src/main/coffee/component-viewer/mixins/main-issues.coffee
@@ -85,7 +85,7 @@ define [
         p = predicate
         predicate = (issue) =>
           creationDate = new Date moment(issue.creationDate).format()
-          (creationDate >= period.get('sinceDate')) && p issue
+          (creationDate > period.get('sinceDate')) && p issue
 
       if requestIssues && !@state.get 'hasIssues'
         @requestIssues(@key).done => @_filterByIssues(predicate)
diff --git a/server/sonar-web/src/main/coffee/component-viewer/source.coffee b/server/sonar-web/src/main/coffee/component-viewer/source.coffee
index 42ae5c05719..079566287a6 100644
--- a/server/sonar-web/src/main/coffee/component-viewer/source.coffee
+++ b/server/sonar-web/src/main/coffee/component-viewer/source.coffee
@@ -24,7 +24,6 @@ define [
   $ = jQuery
 
   API_COVERAGE_TESTS = "#{baseUrl}/api/tests/test_cases"
-  LINES_LIMIT = 3000
   ISSUES_LIMIT = 100
 
 
@@ -104,7 +103,6 @@ define [
         $(expand).insertBefore rows.first()
 
       lines = _.size @model.get 'source'
-      lines = Math.min lines, LINES_LIMIT
       lastShown = rows.last().data('line-number')
       if lastShown < lines
         expand = @expandTemplate
@@ -311,17 +309,16 @@ define [
       source.forEach (sourceLine) =>
         show = false
         line = sourceLine.lineNumber
-        if line <= LINES_LIMIT
-          @showBlocks.forEach (block) ->
-            show = true if block.from <= line && block.to >= line
-          _.extend sourceLine, show: show
+        @showBlocks.forEach (block) ->
+          show = true if block.from <= line && block.to >= line
+        _.extend sourceLine, show: show
       source
 
 
     prepareSource: ->
       source = @model.get 'formattedSource'
       if source?
-        _.first @augmentWithShow(source), LINES_LIMIT
+        @augmentWithShow(source)
 
 
     getStatColumnsCount: ->
@@ -359,6 +356,4 @@ define [
       showZeroLine: @showZeroLine()
       issuesLimit: ISSUES_LIMIT
       issuesLimitReached: @model.get('activeIssues')?.length > ISSUES_LIMIT
-      linesLimit: LINES_LIMIT
-      linesLimitReached: _.size(@model.get 'source') > LINES_LIMIT
       baseDuplications: @getBaseDuplications()
diff --git a/server/sonar-web/src/main/hbs/component-viewer/cw-source.hbs b/server/sonar-web/src/main/hbs/component-viewer/cw-source.hbs
index dbb80fcf7cf..7fb13a6bee8 100644
--- a/server/sonar-web/src/main/hbs/component-viewer/cw-source.hbs
+++ b/server/sonar-web/src/main/hbs/component-viewer/cw-source.hbs
@@ -4,10 +4,6 @@
 
 {{else}}
 
-  {{#if linesLimitReached}}
-    <p class="message-alert marginbottom10">{{tp 'component_viewer.lines_limit_reached' linesLimit}}</p>
-  {{/if}}
-
   {{#if issuesLimitReached}}
     <p class="message-alert marginbottom10">{{tp 'component_viewer.issues_limit_reached' issuesLimit}}</p>
   {{/if}}
diff --git a/sonar-core/src/main/resources/org/sonar/l10n/core.properties b/sonar-core/src/main/resources/org/sonar/l10n/core.properties
index 1939732aa98..2212c677b15 100644
--- a/sonar-core/src/main/resources/org/sonar/l10n/core.properties
+++ b/sonar-core/src/main/resources/org/sonar/l10n/core.properties
@@ -2787,7 +2787,6 @@ component_viewer.more_actions=More Actions
 component_viewer.new_window=Open in New Window
 component_viewer.get_permalink=Get Permalink
 component_viewer.covered_lines=Covered Lines
-component_viewer.lines_limit_reached=For performance reasons, only the {0} first lines will be displayed.
 component_viewer.issues_limit_reached=For usability reasons, only the {0} first issues will be fully displayed. Remaining issues will simply be underlined.
 component_viewer.issues_limit_reached_tooltip={0}\n\nRefine your filter to be able to see the details of this issue.
 component_viewer.cannot_show=We're sorry, but something went wrong. Please try back in a few minutes and contact support if the problem persists.