From ece4346b3e8165d48ea7a064902bc01df2081c24 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Tue, 10 May 2016 17:45:56 +0200 Subject: [PATCH] SONAR-7571 support huge file source in FILE_SOURCES.BINARY_DATA --- .../org/sonar/db/source/FileSourceDto.java | 32 ++++++++++------- .../sonar/db/source/FileSourceDtoTest.java | 35 ++++++++++++++++--- .../v51/FeedFileSourcesBinaryDataTest.java | 4 +-- 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java b/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java index bf568bb1c90..cddfe8957e5 100644 --- a/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java +++ b/sonar-db/src/main/java/org/sonar/db/source/FileSourceDto.java @@ -19,6 +19,8 @@ */ package org.sonar.db.source; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.InvalidProtocolBufferException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -36,6 +38,9 @@ import static java.lang.String.format; public class FileSourceDto { + private static final String SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE = "Protocol message was too large. May be malicious. " + + "Use CodedInputStream.setSizeLimit() to increase the size limit."; + private Long id; private String projectUuid; private String fileUuid; @@ -90,7 +95,7 @@ public class FileSourceDto { public DbFileSources.Data decodeSourceData(byte[] binaryData) { try { - return decodeSourceDataImpl(new ByteArrayInputStream(binaryData)); + return decodeRegularSourceData(binaryData); } catch (IOException e) { throw new IllegalStateException( format("Fail to decompress and deserialize source data [id=%s,fileUuid=%s,projectUuid=%s]", id, fileUuid, projectUuid), @@ -98,21 +103,22 @@ public class FileSourceDto { } } - /** - * Decompress and deserialize content of column FILE_SOURCES.BINARY_DATA. - * The parameter "input" is always closed by this method. - */ - public static DbFileSources.Data decodeSourceData(InputStream binaryInput) { - try { - return decodeSourceDataImpl(binaryInput); - } catch (IOException e) { - throw new IllegalStateException("Fail to decompress and deserialize source data", e); + private static DbFileSources.Data decodeRegularSourceData(byte[] binaryData) throws IOException { + try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) { + return DbFileSources.Data.parseFrom(lz4Input); + } catch (InvalidProtocolBufferException e) { + if (SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE.equals(e.getMessage())) { + return decodeHugeSourceData(binaryData); + } + throw e; } } - private static DbFileSources.Data decodeSourceDataImpl(InputStream binaryInput) throws IOException { - try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(binaryInput)) { - return DbFileSources.Data.parseFrom(lz4Input); + private static DbFileSources.Data decodeHugeSourceData(byte[] binaryData) throws IOException { + try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) { + CodedInputStream input = CodedInputStream.newInstance(lz4Input); + input.setSizeLimit(Integer.MAX_VALUE); + return DbFileSources.Data.parseFrom(input); } } diff --git a/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java b/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java index 0d8dd7b29ee..4b5b0c0798d 100644 --- a/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/source/FileSourceDtoTest.java @@ -29,6 +29,12 @@ import org.sonar.db.protobuf.DbFileSources; import static org.assertj.core.api.Assertions.assertThat; public class FileSourceDtoTest { + private static final String LOREM_IPSUM = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam ac magna libero. " + + "Integer eu quam vulputate, interdum ante quis, sodales mauris. Nam mollis ornare dolor at maximus. Cras pharetra aliquam fringilla. " + + "Nunc hendrerit, elit eu mattis fermentum, ligula metus malesuada nunc, non fermentum augue tellus eu odio. Praesent ut vestibulum nibh. " + + "Curabitur sit amet dignissim magna, at efficitur dolor. Ut non felis aliquam justo euismod gravida. Morbi eleifend vitae ante eu pulvinar. " + + "Aliquam rhoncus magna quis lorem posuere semper."; + @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -54,14 +60,35 @@ public class FileSourceDtoTest { String fileUuid = "file uuid"; String projectUuid = "project uuid"; FileSourceDto underTest = new FileSourceDto() - .setBinaryData(new byte[]{1, 2, 3, 4, 5}) - .setId(id) - .setFileUuid(fileUuid) - .setProjectUuid(projectUuid); + .setBinaryData(new byte[] {1, 2, 3, 4, 5}) + .setId(id) + .setFileUuid(fileUuid) + .setProjectUuid(projectUuid); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Fail to decompress and deserialize source data [id=" + id + ",fileUuid=" + fileUuid + ",projectUuid=" + projectUuid + "]"); underTest.getSourceData(); } + + @Test + public void getSourceData_reads_Data_object_bigger_than_default_size_limit() { + DbFileSources.Data build = createOver64MBDataStructure(); + byte[] bytes = FileSourceDto.encodeSourceData(build); + + DbFileSources.Data data = new FileSourceDto().decodeSourceData(bytes); + assertThat(data.getLinesCount()).isEqualTo(build.getLinesCount()); + } + + private static DbFileSources.Data createOver64MBDataStructure() { + DbFileSources.Data.Builder dataBuilder = DbFileSources.Data.newBuilder(); + DbFileSources.Line.Builder lineBuilder = DbFileSources.Line.newBuilder(); + for (int i = 0; i < 199999; i++) { + dataBuilder.addLines( + lineBuilder.setSource(LOREM_IPSUM) + .setLine(i) + .build()); + } + return dataBuilder.build(); + } } diff --git a/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java b/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java index a3a93e51ca3..74f187617f0 100644 --- a/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java +++ b/sonar-db/src/test/java/org/sonar/db/version/v51/FeedFileSourcesBinaryDataTest.java @@ -19,7 +19,6 @@ */ package org.sonar.db.version.v51; -import java.io.InputStream; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -87,8 +86,7 @@ public class FeedFileSourcesBinaryDataTest { pstmt.setLong(1, fileSourceId); rs = pstmt.executeQuery(); rs.next(); - InputStream data = rs.getBinaryStream(1); - return FileSourceDto.decodeSourceData(data); + return new FileSourceDto().decodeSourceData(rs.getBytes(1)); } finally { DbUtils.closeQuietly(rs); DbUtils.closeQuietly(pstmt); -- 2.39.5