From bd4db3165d091feeb4df6fb9f165f0d5c689c722 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Wed, 26 Nov 2014 23:02:56 +0100 Subject: [PATCH] Support empty files in SourceLineResultSetIterator --- .../server/source/index/SourceLineDoc.java | 2 +- .../index/SourceLineIndexDefinition.java | 4 +- .../source/index/SourceLineIndexer.java | 4 -- .../index/SourceLineResultSetIterator.java | 63 ++++++++++--------- .../server/source/index/package-info.java | 24 +++++++ .../SourceLineResultSetIteratorTest.java | 21 +++++-- .../empty-file.xml | 6 ++ .../{source-with-scm.xml => shared.xml} | 0 8 files changed, 80 insertions(+), 44 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/source/index/package-info.java create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/empty-file.xml rename server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/{source-with-scm.xml => shared.xml} (100%) diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java index 84451822dff..047cc24ea0f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java @@ -103,7 +103,7 @@ public class SourceLineDoc extends BaseDoc { } public Date updateDate() { - return getField(BaseNormalizer.UPDATED_AT_FIELD); + return getFieldAsDate(BaseNormalizer.UPDATED_AT_FIELD); } public void setUpdateDate(Date updatedAt) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java index b1ce5cd981d..fc2eb002ed6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java @@ -39,7 +39,7 @@ public class SourceLineIndexDefinition implements IndexDefinition { public static final String INDEX_SOURCE_LINES = "sourcelines"; - public static final String TYPE_SOURCE_LINE = "sourceLine"; + public static final String TYPE_SOURCE_LINE = "sourceline"; private final Settings settings; @@ -60,7 +60,7 @@ public class SourceLineIndexDefinition implements IndexDefinition { // else keep defaults (one shard) } - // type "sourceLine" + // type "sourceline" NewIndex.NewIndexType sourceLineMapping = index.createType(TYPE_SOURCE_LINE); sourceLineMapping.stringFieldBuilder(FIELD_PROJECT_UUID).build(); sourceLineMapping.stringFieldBuilder(FIELD_FILE_UUID).build(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java index 2895663d2cc..6116ea942d7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java @@ -30,9 +30,6 @@ import java.sql.Connection; import java.util.Collection; import java.util.Iterator; -/** - * Not thread-safe - */ public class SourceLineIndexer implements ServerComponent { private final DbClient dbClient; @@ -47,7 +44,6 @@ public class SourceLineIndexer implements ServerComponent { } public void indexSourceLines(boolean large) { - // TODO support timezones final BulkIndexer bulk = new BulkIndexer(esClient, SourceLineIndexDefinition.INDEX_SOURCE_LINES); bulk.setLarge(large); diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java index 94ebaf9ad24..0bafa2fc512 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java @@ -51,10 +51,8 @@ class SourceLineResultSetIterator extends ResultSetIterator read(ResultSet rs) throws SQLException { String projectUuid = rs.getString(1); String fileUuid = rs.getString(2); - Date updatedAt = new Date(SqlUtil.getLong(rs, 4)); + Date updatedAt = new Date(SqlUtil.getLong(rs, 3)); + String csv = rs.getString(4); - int line = 1; List lines = Lists.newArrayList(); - CSVParser csvParser = null; - try { - csvParser = new CSVParser(new StringReader(rs.getString(5)), CSVFormat.DEFAULT); + if (StringUtils.isNotEmpty(csv)) { + int line = 1; + CSVParser csvParser = null; + try { + csvParser = new CSVParser(new StringReader(csv), CSVFormat.DEFAULT); - for(CSVRecord csvRecord: csvParser) { - SourceLineDoc doc = new SourceLineDoc(Maps.newHashMapWithExpectedSize(9)); + for (CSVRecord csvRecord : csvParser) { + SourceLineDoc doc = new SourceLineDoc(Maps.newHashMapWithExpectedSize(9)); - doc.setProjectUuid(projectUuid); - doc.setFileUuid(fileUuid); - doc.setLine(line); - doc.setUpdateDate(updatedAt); - doc.setScmRevision(csvRecord.get(0)); - doc.setScmAuthor(csvRecord.get(1)); - doc.setScmDate(DateUtils.parseDateTimeQuietly(csvRecord.get(2))); - // doc.setLineHits(csvRecord.get(3)); - // doc.setConditions(csvRecord.get(4)); - // doc.setCoveredConditions(csvRecord.get(5)); - doc.setHighlighting(csvRecord.get(6)); - doc.setSource(csvRecord.get(csvRecord.size() - 1)); + doc.setProjectUuid(projectUuid); + doc.setFileUuid(fileUuid); + doc.setLine(line); + doc.setUpdateDate(updatedAt); + doc.setScmRevision(csvRecord.get(0)); + doc.setScmAuthor(csvRecord.get(1)); + doc.setScmDate(DateUtils.parseDateTimeQuietly(csvRecord.get(2))); + // doc.setLineHits(csvRecord.get(3)); + // doc.setConditions(csvRecord.get(4)); + // doc.setCoveredConditions(csvRecord.get(5)); + doc.setHighlighting(csvRecord.get(6)); + doc.setSource(csvRecord.get(csvRecord.size() - 1)); - lines.add(doc); + lines.add(doc); - line ++; + line++; + } + } catch (IOException ioError) { + throw new IllegalStateException("Impossible to open stream for file_sources.data with file_uuid " + fileUuid); + } catch (ArrayIndexOutOfBoundsException lineError) { + throw new IllegalStateException( + String.format("Impossible to parse source line data, stuck at line %d", line), lineError); + } finally { + IOUtils.closeQuietly(csvParser); } - } catch(IOException ioError) { - throw new IllegalStateException("Impossible to open stream for file_sources.data with file_uuid " + fileUuid); - } catch(ArrayIndexOutOfBoundsException lineError) { - throw new IllegalStateException( - String.format("Impossible to parse source line data, stuck at line %d", line), lineError); - } finally { - IOUtils.closeQuietly(csvParser); } return lines; diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/package-info.java new file mode 100644 index 00000000000..78d3dff872d --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/package-info.java @@ -0,0 +1,24 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +@ParametersAreNonnullByDefault +package org.sonar.server.source.index; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java index b79636e89c2..57ffb0f6e4d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java @@ -24,12 +24,12 @@ import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; -import org.sonar.api.utils.DateUtils; import org.sonar.core.persistence.TestDatabase; import org.sonar.server.db.DbClient; import java.sql.Connection; import java.sql.PreparedStatement; +import java.util.Collection; import java.util.List; import static org.fest.assertions.Assertions.assertThat; @@ -58,7 +58,7 @@ public class SourceLineResultSetIteratorTest { @Test public void should_generate_source_line_documents() throws Exception { - db.prepareDbUnit(getClass(), "source-with-scm.xml"); + db.prepareDbUnit(getClass(), "shared.xml"); PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET data = ? WHERE id=1"); stmt.setString(1, "aef12a,alice,2014-04-25T12:34:56+0100,,,,polop,class Foo {\r\n" + "abe465,bob,2014-07-25T12:34:56+0100,,,,, // Empty\r\n" + @@ -84,16 +84,26 @@ public class SourceLineResultSetIteratorTest { @Test public void should_ignore_lines_already_handled() throws Exception { - db.prepareDbUnit(getClass(), "source-with-scm.xml"); + db.prepareDbUnit(getClass(), "shared.xml"); SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, db.openConnection(), - DateUtils.parseDateTime("2014-11-17T16:44:02+0100").getTime()); + 2000000000000L); assertThat(iterator.hasNext()).isFalse(); } + @Test + public void parse_empty_file() throws Exception { + db.prepareDbUnit(getClass(), "empty-file.xml"); + + SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, db.openConnection(), 0L); + assertThat(iterator.hasNext()).isTrue(); + Collection lines = iterator.next(); + assertThat(lines).isEmpty(); + } + @Test public void should_fail_on_bad_csv() throws Exception { - db.prepareDbUnit(getClass(), "source-with-scm.xml"); + db.prepareDbUnit(getClass(), "shared.xml"); PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET data = ? WHERE id=1"); stmt.setString(1, "plouf"); stmt.executeUpdate(); @@ -108,7 +118,6 @@ public class SourceLineResultSetIteratorTest { // ok } finally { iterator.close(); - } } } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/empty-file.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/empty-file.xml new file mode 100644 index 00000000000..83fcc118164 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/empty-file.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/shared.xml similarity index 100% rename from server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml rename to server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/shared.xml -- 2.39.5