SONAR-7571 support huge file source in FILE_SOURCES.BINARY_DATA

diff --git a/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java b/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java
index bf568bb1c90206042cb47f8b81a54b4aef233308..cddfe8957e53e0489b6ac1f1dcd0adc653e3cf37 100644 (file)
--- a/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java
+++ b/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java
@@ -19,6 +19,8 @@
  */
 package org.sonar.db.source;
 
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.InvalidProtocolBufferException;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -36,6 +38,9 @@ import static java.lang.String.format;
 
 public class FileSourceDto {
 
+  private static final String SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE = "Protocol message was too large.  May be malicious.  " +
+    "Use CodedInputStream.setSizeLimit() to increase the size limit.";
+
   private Long id;
   private String projectUuid;
   private String fileUuid;
@@ -90,7 +95,7 @@ public class FileSourceDto {
 
   public DbFileSources.Data decodeSourceData(byte[] binaryData) {
     try {
-      return decodeSourceDataImpl(new ByteArrayInputStream(binaryData));
+      return decodeRegularSourceData(binaryData);
     } catch (IOException e) {
       throw new IllegalStateException(
         format("Fail to decompress and deserialize source data [id=%s,fileUuid=%s,projectUuid=%s]", id, fileUuid, projectUuid),
@@ -98,21 +103,22 @@ public class FileSourceDto {
     }
   }
 
-  /**
-   * Decompress and deserialize content of column FILE_SOURCES.BINARY_DATA.
-   * The parameter "input" is always closed by this method.
-   */
-  public static DbFileSources.Data decodeSourceData(InputStream binaryInput) {
-    try {
-      return decodeSourceDataImpl(binaryInput);
-    } catch (IOException e) {
-      throw new IllegalStateException("Fail to decompress and deserialize source data", e);
+  private static DbFileSources.Data decodeRegularSourceData(byte[] binaryData) throws IOException {
+    try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) {
+      return DbFileSources.Data.parseFrom(lz4Input);
+    } catch (InvalidProtocolBufferException e) {
+      if (SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE.equals(e.getMessage())) {
+        return decodeHugeSourceData(binaryData);
+      }
+      throw e;
     }
   }
 
-  private static DbFileSources.Data decodeSourceDataImpl(InputStream binaryInput) throws IOException {
-    try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(binaryInput)) {
-      return DbFileSources.Data.parseFrom(lz4Input);
+  private static DbFileSources.Data decodeHugeSourceData(byte[] binaryData) throws IOException {
+    try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) {
+      CodedInputStream input = CodedInputStream.newInstance(lz4Input);
+      input.setSizeLimit(Integer.MAX_VALUE);
+      return DbFileSources.Data.parseFrom(input);
     }
   }
 

diff --git a/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java b/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java
index 0d8dd7b29eef3d16ba6da7ad055742b50655ffb1..4b5b0c0798d60c5277aa193100ec32e26dc62acd 100644 (file)
--- a/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java
+++ b/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java
@@ -29,6 +29,12 @@ import org.sonar.db.protobuf.DbFileSources;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class FileSourceDtoTest {
+  private static final String LOREM_IPSUM = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam ac magna libero. " +
+    "Integer eu quam vulputate, interdum ante quis, sodales mauris. Nam mollis ornare dolor at maximus. Cras pharetra aliquam fringilla. " +
+    "Nunc hendrerit, elit eu mattis fermentum, ligula metus malesuada nunc, non fermentum augue tellus eu odio. Praesent ut vestibulum nibh. " +
+    "Curabitur sit amet dignissim magna, at efficitur dolor. Ut non felis aliquam justo euismod gravida. Morbi eleifend vitae ante eu pulvinar. " +
+    "Aliquam rhoncus magna quis lorem posuere semper.";
+
   @Rule
   public ExpectedException expectedException = ExpectedException.none();
 
@@ -54,14 +60,35 @@ public class FileSourceDtoTest {
     String fileUuid = "file uuid";
     String projectUuid = "project uuid";
     FileSourceDto underTest = new FileSourceDto()
-        .setBinaryData(new byte[]{1, 2, 3, 4, 5})
-        .setId(id)
-        .setFileUuid(fileUuid)
-        .setProjectUuid(projectUuid);
+      .setBinaryData(new byte[] {1, 2, 3, 4, 5})
+      .setId(id)
+      .setFileUuid(fileUuid)
+      .setProjectUuid(projectUuid);
 
     expectedException.expect(IllegalStateException.class);
     expectedException.expectMessage("Fail to decompress and deserialize source data [id=" + id + ",fileUuid=" + fileUuid + ",projectUuid=" + projectUuid + "]");
 
     underTest.getSourceData();
   }
+
+  @Test
+  public void getSourceData_reads_Data_object_bigger_than_default_size_limit() {
+    DbFileSources.Data build = createOver64MBDataStructure();
+    byte[] bytes = FileSourceDto.encodeSourceData(build);
+
+    DbFileSources.Data data = new FileSourceDto().decodeSourceData(bytes);
+    assertThat(data.getLinesCount()).isEqualTo(build.getLinesCount());
+  }
+
+  private static DbFileSources.Data createOver64MBDataStructure() {
+    DbFileSources.Data.Builder dataBuilder = DbFileSources.Data.newBuilder();
+    DbFileSources.Line.Builder lineBuilder = DbFileSources.Line.newBuilder();
+    for (int i = 0; i < 199999; i++) {
+      dataBuilder.addLines(
+        lineBuilder.setSource(LOREM_IPSUM)
+          .setLine(i)
+          .build());
+    }
+    return dataBuilder.build();
+  }
 }

diff --git a/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java b/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java
index a3a93e51ca38b17ff3f37244be17f63d313ee97a..74f187617f045468750c88274c3063d0c5fa5f54 100644 (file)
--- a/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java
+++ b/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java
@@ -19,7 +19,6 @@
  */
 package org.sonar.db.version.v51;
 
-import java.io.InputStream;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
@@ -87,8 +86,7 @@ public class FeedFileSourcesBinaryDataTest {
       pstmt.setLong(1, fileSourceId);
       rs = pstmt.executeQuery();
       rs.next();
-      InputStream data = rs.getBinaryStream(1);
-      return FileSourceDto.decodeSourceData(data);
+      return new FileSourceDto().decodeSourceData(rs.getBytes(1));
     } finally {
       DbUtils.closeQuietly(rs);
       DbUtils.closeQuietly(pstmt);