From 984f87ae7fcb870dfdb33a9bf71bab456757aefa Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Tue, 2 Dec 2014 10:08:34 +0100 Subject: [PATCH] Scroll FILE_SOURCES rows one by one --- .../main/java/org/sonar/server/db/DbClient.java | 16 +++++++++++++++- .../sonar/server/source/index/SourceLineDoc.java | 16 ++++++++-------- .../index/SourceLineResultSetIterator.java | 5 ++++- .../persistence/dialect/AbstractDialect.java | 5 +++++ .../sonar/core/persistence/dialect/Dialect.java | 8 ++++++++ .../sonar/core/persistence/dialect/MySql.java | 5 +++++ .../core/persistence/dialect/MySqlTest.java | 1 + .../core/persistence/dialect/OracleTest.java | 1 + 8 files changed, 47 insertions(+), 10 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java b/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java index 248eb8dbac8..e40789d9a81 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java @@ -220,9 +220,23 @@ public class DbClient implements ServerComponent { * Create a PreparedStatement for SELECT requests with scrolling of results */ public final PreparedStatement newScrollingSelectStatement(Connection connection, String sql) { + int fetchSize = database().getDialect().getScrollDefaultFetchSize(); + return newScrollingSelectStatement(connection, sql, fetchSize); + } + + /** + * Create a PreparedStatement for SELECT requests with scrolling of results row by row (only one row + * in memory at a time) + */ + public final PreparedStatement newScrollingSingleRowSelectStatement(Connection connection, String sql) { + int fetchSize = database().getDialect().getScrollSingleRowFetchSize(); + return newScrollingSelectStatement(connection, sql, fetchSize); + } + + private PreparedStatement newScrollingSelectStatement(Connection connection, String sql, int fetchSize) { try { PreparedStatement stmt = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - stmt.setFetchSize(database().getDialect().getScrollDefaultFetchSize()); + stmt.setFetchSize(fetchSize); return stmt; } catch (SQLException e) { throw new IllegalStateException("Fail to create SQL statement: " + sql, e); diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java index f7235215a53..5c9c1123fb5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java @@ -119,7 +119,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer utLineHits() { - Number lineHits = (Number)getNullableField(SourceLineIndexDefinition.FIELD_UT_LINE_HITS); + Number lineHits = getNullableField(SourceLineIndexDefinition.FIELD_UT_LINE_HITS); return lineHits == null ? null : lineHits.intValue(); } @@ -129,7 +129,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer utConditions() { - Number conditions = (Number)getNullableField(SourceLineIndexDefinition.FIELD_UT_CONDITIONS); + Number conditions = getNullableField(SourceLineIndexDefinition.FIELD_UT_CONDITIONS); return conditions == null ? null : conditions.intValue(); } @@ -139,7 +139,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer utCoveredConditions() { - Number coveredConditions = (Number)getNullableField(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS); + Number coveredConditions = getNullableField(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS); return coveredConditions == null ? null : coveredConditions.intValue(); } @@ -149,7 +149,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer itLineHits() { - Number lineHits = (Number)getNullableField(SourceLineIndexDefinition.FIELD_IT_LINE_HITS); + Number lineHits = getNullableField(SourceLineIndexDefinition.FIELD_IT_LINE_HITS); return lineHits == null ? null : lineHits.intValue(); } @@ -159,7 +159,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer itConditions() { - Number conditions = (Number)getNullableField(SourceLineIndexDefinition.FIELD_IT_CONDITIONS); + Number conditions = getNullableField(SourceLineIndexDefinition.FIELD_IT_CONDITIONS); return conditions == null ? null : conditions.intValue(); } @@ -179,7 +179,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer overallLineHits() { - Number lineHits = (Number)getNullableField(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS); + Number lineHits = getNullableField(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS); return lineHits == null ? null : lineHits.intValue(); } @@ -189,7 +189,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer overallConditions() { - Number conditions = (Number)getNullableField(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS); + Number conditions = getNullableField(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS); return conditions == null ? null : conditions.intValue(); } @@ -199,7 +199,7 @@ public class SourceLineDoc extends BaseDoc { @CheckForNull public Integer overallCoveredConditions() { - Number coveredConditions = (Number)getNullableField(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS); + Number coveredConditions = getNullableField(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS); return coveredConditions == null ? null : coveredConditions.intValue(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java index ac0c80311a4..9aed63d6bb3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java @@ -32,6 +32,7 @@ import org.sonar.server.db.DbClient; import org.sonar.server.db.ResultSetIterator; import org.sonar.server.db.migrations.SqlUtil; +import javax.annotation.CheckForNull; import javax.annotation.Nullable; import java.io.IOException; @@ -91,7 +92,8 @@ public class SourceLineResultSetIterator extends ResultSetIterator 0L ? SQL_AFTER_DATE : SQL_ALL; - PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql); + // rows are big, so they are scrolled once at a time (one row in memory at a time) + PreparedStatement stmt = dbClient.newScrollingSingleRowSelectStatement(connection, sql); if (afterDate > 0L) { stmt.setLong(1, afterDate); } @@ -179,6 +181,7 @@ public class SourceLineResultSetIterator extends ResultSetIterator