From 41dc465ea47931964d4b3a28a58cd8a9e35099d8 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lievremont Date: Thu, 20 Nov 2014 17:01:47 +0100 Subject: [PATCH] SONAR-5801 Index source lines at startup and after analysis --- .../computation/ComputationStepRegistry.java | 3 + .../java/org/sonar/server/es/BulkIndexer.java | 1 + .../server/platform/ServerComponents.java | 6 +- .../server/search/IndexSynchronizer.java | 9 +- .../server/source/IndexSourceLinesStep.java | 46 ++++++++ .../server/source/index/SourceLineDoc.java | 2 +- .../index/SourceLineIndexDefinition.java | 6 +- .../source/index/SourceLineIndexer.java | 3 +- .../index/SourceLineResultSetIterator.java | 37 +++--- .../ComputationStepRegistryMediumTest.java | 5 +- .../search/IndexSynchronizerMediumTest.java | 5 +- .../source/index/SourceLineIndexerTest.java | 75 ++++++++++++ .../SourceLineResultSetIteratorTest.java | 109 ++++++++++++++++++ .../schema.sql | 10 ++ .../source-with-scm.xml | 6 + 15 files changed, 294 insertions(+), 29 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java b/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java index 0049337f07d..2d6b26b67ca 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/ComputationStepRegistry.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.sonar.api.ServerComponent; import org.sonar.api.platform.ComponentContainer; +import org.sonar.server.source.IndexSourceLinesStep; import java.util.List; @@ -50,6 +51,8 @@ public class ComputationStepRegistry implements ServerComponent { steps.add(pico.getComponentByType(DataCleanerStep.class)); // project only steps.add(pico.getComponentByType(IndexProjectIssuesStep.class)); + // project only + steps.add(pico.getComponentByType(IndexSourceLinesStep.class)); return ImmutableList.copyOf(steps); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index 07eb4bb8a82..645130ffa58 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -112,6 +112,7 @@ public class BulkIndexer implements Startable { bulkRequest.request().add(request); if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) { executeBulk(bulkRequest); + bulkRequest = client.prepareBulk(); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index dd0a25b7c1b..2dc9b64f54d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -158,10 +158,7 @@ import org.sonar.server.rule.index.RuleIndex; import org.sonar.server.rule.index.RuleNormalizer; import org.sonar.server.rule.ws.*; import org.sonar.server.search.*; -import org.sonar.server.source.CodeColorizers; -import org.sonar.server.source.DeprecatedSourceDecorator; -import org.sonar.server.source.HtmlSourceDecorator; -import org.sonar.server.source.SourceService; +import org.sonar.server.source.*; import org.sonar.server.source.index.SourceLineIndexDefinition; import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.source.ws.*; @@ -552,6 +549,7 @@ class ServerComponents { pico.addSingleton(ScmAction.class); pico.addSingleton(SourceLineIndexDefinition.class); pico.addSingleton(SourceLineIndexer.class); + pico.addSingleton(IndexSourceLinesStep.class); // Duplications pico.addSingleton(DuplicationsParser.class); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index 462c9215bdd..b0bece7d810 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -32,6 +32,7 @@ import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.issue.index.IssueNormalizer; import org.sonar.server.qualityprofile.index.ActiveRuleIndex; import org.sonar.server.rule.index.RuleIndex; +import org.sonar.server.source.index.SourceLineIndexer; import java.util.Date; import java.util.List; @@ -46,10 +47,12 @@ public class IndexSynchronizer { private final DbClient db; private final IndexClient index; + private final SourceLineIndexer sourceLineIndexer; - public IndexSynchronizer(DbClient db, IndexClient index) { + public IndexSynchronizer(DbClient db, IndexClient index, SourceLineIndexer sourceLineIndexer) { this.db = db; this.index = index; + this.sourceLineIndexer = sourceLineIndexer; } public void execute() { @@ -66,6 +69,10 @@ public class IndexSynchronizer { IssueAuthorizationNormalizer.IssueAuthorizationField.PROJECT.field(), projectUuids); synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class)); synchronize(session, db.activityDao(), index.get(ActivityIndex.class)); + + LOG.info("Indexing of sourceLine records"); + sourceLineIndexer.indexSourceLines(true); + session.commit(); LOG.info("Synchronization done in {}ms...", System.currentTimeMillis() - start); } finally { diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java b/server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java new file mode 100644 index 00000000000..ea842928e5b --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java @@ -0,0 +1,46 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.source; + +import org.sonar.core.component.ComponentDto; +import org.sonar.core.computation.db.AnalysisReportDto; +import org.sonar.core.persistence.DbSession; +import org.sonar.server.computation.ComputationStep; +import org.sonar.server.source.index.SourceLineIndexer; + +public class IndexSourceLinesStep implements ComputationStep { + + private SourceLineIndexer indexer; + + public IndexSourceLinesStep(SourceLineIndexer indexer) { + this.indexer = indexer; + } + + @Override + public void execute(DbSession session, AnalysisReportDto analysisReportDto, ComponentDto project) { + indexer.indexSourceLines(false); + } + + @Override + public String getDescription() { + return "Put source code into search index"; + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java index 225aa2648ae..1ece35e3258 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java @@ -84,7 +84,7 @@ public class SourceLineDoc extends BaseDoc { } public void setHighlighting(String fileUuid) { - setField(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid); + setField(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, fileUuid); } public String source() { diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java index c473e62a083..b1ce5cd981d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java @@ -31,9 +31,9 @@ public class SourceLineIndexDefinition implements IndexDefinition { public static final String FIELD_PROJECT_UUID = "projectUuid"; public static final String FIELD_FILE_UUID = "fileUuid"; public static final String FIELD_LINE = "line"; - public static final String FIELD_SCM_REVISION = "scm_revision"; - public static final String FIELD_SCM_AUTHOR = "scm_author"; - public static final String FIELD_SCM_DATE = "scm_date"; + public static final String FIELD_SCM_REVISION = "scmRevision"; + public static final String FIELD_SCM_AUTHOR = "scmAuthor"; + public static final String FIELD_SCM_DATE = "scmDate"; public static final String FIELD_HIGHLIGHTING = "highlighting"; public static final String FIELD_SOURCE = "source"; diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java index 27227fad08e..8dfa2cedf2a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java @@ -25,7 +25,6 @@ import org.sonar.core.persistence.DbSession; import org.sonar.server.db.DbClient; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; -import org.sonar.server.es.IssueIndexDefinition; import java.sql.Connection; import java.util.Collection; @@ -92,7 +91,7 @@ public class SourceLineIndexer implements ServerComponent { private UpdateRequest newUpsertRequest(SourceLineDoc lineDoc) { String projectUuid = lineDoc.projectUuid(); - return new UpdateRequest(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE, lineDoc.key()) + return new UpdateRequest(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE, lineDoc.key()) .routing(projectUuid) .doc(lineDoc.getFields()) .upsert(lineDoc.getFields()); diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java index 47cb2ef5bd4..e315e83b0b4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java @@ -31,7 +31,9 @@ import org.sonar.server.db.DbClient; import org.sonar.server.db.ResultSetIterator; import org.sonar.server.db.migrations.SqlUtil; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.io.Reader; import java.sql.*; import java.util.Collection; @@ -56,18 +58,18 @@ class SourceLineResultSetIterator extends ResultSetIterator=?"; + private static final String SQL_AFTER_DATE = SQL_ALL + " where updated_at>=?"; static SourceLineResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) { try { String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL; PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql); if (afterDate > 0L) { - stmt.setTimestamp(0, new Timestamp(afterDate)); + stmt.setTimestamp(1, new Timestamp(afterDate)); } return new SourceLineResultSetIterator(stmt); } catch (SQLException e) { - throw new IllegalStateException("Fail to prepare SQL request to select all issues", e); + throw new IllegalStateException("Fail to prepare SQL request to select all file sources", e); } } @@ -77,13 +79,10 @@ class SourceLineResultSetIterator extends ResultSetIterator read(ResultSet rs) throws SQLException { - String projectUuid = rs.getString(1); String fileUuid = rs.getString(2); - // createdAt = rs.getDate(3); Date updatedAt = SqlUtil.getDate(rs, 4); - Reader dataStream = rs.getClob(5).getCharacterStream(); - // String dataHash = rs.getString(6); + Reader dataStream = new InputStreamReader(new ByteArrayInputStream(rs.getBytes(5))); int line = 1; List lines = Lists.newArrayList(); @@ -91,22 +90,28 @@ class SourceLineResultSetIterator extends ResultSetIteratornewHashMapWithExpectedSize(8)); + for(CSVRecord csvRecord: csvParser) { + SourceLineDoc doc = new SourceLineDoc(Maps.newHashMapWithExpectedSize(9)); doc.setProjectUuid(projectUuid); doc.setFileUuid(fileUuid); - doc.setLine(line ++); + doc.setLine(line); doc.setUpdateDate(updatedAt); - doc.setScmRevision(record.get(0)); - doc.setScmAuthor(record.get(1)); - doc.setScmDate(DateUtils.parseDateTimeQuietly(record.get(2))); - doc.setHighlighting(record.get(3)); - doc.setSource(record.get(4)); + doc.setScmRevision(csvRecord.get(0)); + doc.setScmAuthor(csvRecord.get(1)); + doc.setScmDate(DateUtils.parseDateTimeQuietly(csvRecord.get(2))); + doc.setHighlighting(csvRecord.get(3)); + doc.setSource(csvRecord.get(4)); + + lines.add(doc); + + line ++; } } catch(IOException ioError) { + throw new IllegalStateException("Impossible to open stream for file_sources.data with file_uuid " + fileUuid); + } catch(ArrayIndexOutOfBoundsException lineError) { throw new IllegalStateException( - String.format("Impossible to parse source line data, stuck at line %d", line), ioError); + String.format("Impossible to parse source line data, stuck at line %d", line), lineError); } finally { IOUtils.closeQuietly(csvParser); IOUtils.closeQuietly(dataStream); diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryMediumTest.java index e059c099fbf..c9085d3d9d4 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/ComputationStepRegistryMediumTest.java @@ -25,6 +25,7 @@ import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.sonar.api.platform.ComponentContainer; +import org.sonar.server.source.IndexSourceLinesStep; import org.sonar.server.tester.ServerTester; import java.util.List; @@ -49,6 +50,7 @@ public class ComputationStepRegistryMediumTest { pico.addSingleton(mock(DataCleanerStep.class)); pico.addSingleton(mock(InvalidatePreviewCacheStep.class)); pico.addSingleton(mock(ComponentIndexationInDatabaseStep.class)); + pico.addSingleton(mock(IndexSourceLinesStep.class)); sut = new ComputationStepRegistry(pico); } @@ -61,7 +63,8 @@ public class ComputationStepRegistryMediumTest { InvalidatePreviewCacheStep.class, ComponentIndexationInDatabaseStep.class, DataCleanerStep.class, - IndexProjectIssuesStep.class + IndexProjectIssuesStep.class, + IndexSourceLinesStep.class ); List steps = sut.steps(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java index 2e9569dc456..9766afbef76 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/IndexSynchronizerMediumTest.java @@ -39,6 +39,7 @@ import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.rule.RuleTesting; import org.sonar.server.rule.db.RuleDao; import org.sonar.server.rule.index.RuleIndex; +import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.tester.ServerTester; import static org.fest.assertions.Assertions.assertThat; @@ -51,14 +52,16 @@ public class IndexSynchronizerMediumTest { IndexSynchronizer synchronizer; DbClient dbClient; IndexClient indexClient; + SourceLineIndexer sourceLineIndexer; DbSession dbSession; @Before public void setUp() throws Exception { dbClient = tester.get(DbClient.class); indexClient = tester.get(IndexClient.class); + sourceLineIndexer = tester.get(SourceLineIndexer.class); dbSession = dbClient.openSession(false); - synchronizer = new IndexSynchronizer(dbClient, indexClient); + synchronizer = new IndexSynchronizer(dbClient, indexClient, sourceLineIndexer); tester.clearDbAndIndexes(); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java new file mode 100644 index 00000000000..3cc23526c6f --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java @@ -0,0 +1,75 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.source.index; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.server.db.DbClient; +import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.EsTester; +import org.sonar.server.search.BaseNormalizer; +import org.sonar.server.source.index.SourceLineDoc; +import org.sonar.server.source.index.SourceLineIndexDefinition; +import org.sonar.server.source.index.SourceLineIndexer; + +import java.util.Collection; +import java.util.Date; +import java.util.List; + +import static org.mockito.Mockito.mock; + +public class SourceLineIndexerTest { + + @Rule + public EsTester es = new EsTester().addDefinitions(new SourceLineIndexDefinition(new Settings())); + + private SourceLineIndexer indexer; + + @Before + public void setUp() { + indexer = new SourceLineIndexer(mock(DbClient.class), es.client()); + } + + @Test + public void should_index_source_lines() { + BulkIndexer bulk = new BulkIndexer(es.client(), SourceLineIndexDefinition.INDEX_SOURCE_LINES); + + SourceLineDoc line1 = new SourceLineDoc(ImmutableMap.builder() + .put(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "abcd") + .put(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh") + .put(SourceLineIndexDefinition.FIELD_LINE, 42) + .put(SourceLineIndexDefinition.FIELD_SCM_REVISION, "cafebabe") + .put(SourceLineIndexDefinition.FIELD_SCM_DATE, "2014-01-01T12:34:56.7+0100") + .put(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "polop") + .put(SourceLineIndexDefinition.FIELD_SOURCE, "package org.sonar.server.source;") + .put(BaseNormalizer.UPDATED_AT_FIELD, new Date()) + .build()); + Collection sourceLines = ImmutableList.of(line1); + + List> sourceLineContainer = Lists.newArrayList(); + sourceLineContainer.add(sourceLines); + indexer.indexSourceLines(bulk, sourceLineContainer.iterator()); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java new file mode 100644 index 00000000000..c299d6e386f --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java @@ -0,0 +1,109 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.source.index; + +import com.google.common.collect.Lists; +import org.apache.commons.io.Charsets; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.sonar.api.utils.DateUtils; +import org.sonar.core.persistence.TestDatabase; +import org.sonar.server.db.DbClient; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.List; + +import static org.fest.assertions.Assertions.assertThat; + +public class SourceLineResultSetIteratorTest { + + @ClassRule + public static TestDatabase db = new TestDatabase(); + + DbClient dbClient; + + Connection connection; + + @Before + public void setUp() throws Exception { + dbClient = new DbClient(db.database(), db.myBatis()); + db.schema(this.getClass(), "schema.sql"); + connection = db.openConnection(); + } + + @After + public void after() throws Exception { + connection.close(); + } + + @Test + public void should_generate_source_line_documents() throws Exception { + db.prepareDbUnit(getClass(), "source-with-scm.xml"); + Connection connection = db.openConnection(); + PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET data = ? WHERE id=1"); + stmt.setBytes(1, ("aef12a,alice,2014-04-25T12:34:56+0100,,class Foo {\r\n" + + "abe465,bob,2014-07-25T12:34:56+0100,, // Empty\r\n" + + "afb789,carol,2014-03-23T12:34:56+0100,,}\r\n" + + "afb789,carol,2014-03-23T12:34:56+0100,,\r\n").getBytes(Charsets.UTF_8)); + stmt.executeUpdate(); + connection.commit(); + + SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L); + assertThat(iterator.hasNext()).isTrue(); + List sourceLines = Lists.newArrayList(iterator.next()); + assertThat(sourceLines).hasSize(4); + SourceLineDoc firstLine = sourceLines.get(0); + assertThat(firstLine.projectUuid()).isEqualTo("uuid-MyProject"); + assertThat(firstLine.fileUuid()).isEqualTo("uuid-MyFile.xoo"); + assertThat(firstLine.line()).isEqualTo(1); + assertThat(firstLine.scmRevision()).isEqualTo("aef12a"); + assertThat(firstLine.scmAuthor()).isEqualTo("alice"); + assertThat(firstLine.scmDate()).isEqualTo(DateUtils.parseDateTime("2014-04-25T12:34:56+0100")); + assertThat(firstLine.highlighting()).isEmpty(); + assertThat(firstLine.source()).isEqualTo("class Foo {"); + } + + @Test + public void should_ignore_lines_already_handled() throws Exception { + db.prepareDbUnit(getClass(), "source-with-scm.xml"); + + SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, db.openConnection(), + DateUtils.parseDateTime("2014-11-01T16:44:02+0100").getTime()); + assertThat(iterator.hasNext()).isFalse(); + } + + @Test(expected = IllegalStateException.class) + public void should_fail_on_bad_csv() throws Exception { + db.prepareDbUnit(getClass(), "source-with-scm.xml"); + Connection connection = db.openConnection(); + PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET data = ? WHERE id=1"); + stmt.setBytes(1, ("plouf").getBytes(Charsets.UTF_8)); + stmt.executeUpdate(); + connection.commit(); + + SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L); + assertThat(iterator.hasNext()).isTrue(); + iterator.next(); + } + +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql new file mode 100644 index 00000000000..d045f1f012b --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql @@ -0,0 +1,10 @@ + +CREATE TABLE "FILE_SOURCES" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "PROJECT_UUID" VARCHAR(50) NOT NULL, + "FILE_UUID" VARCHAR(50) NOT NULL, + "DATA" CLOB(2147483647), + "DATA_HASH" VARCHAR(50) NOT NULL, + "CREATED_AT" TIMESTAMP NOT NULL, + "UPDATED_AT" TIMESTAMP NOT NULL +); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml new file mode 100644 index 00000000000..3f2161acff0 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml @@ -0,0 +1,6 @@ + + + + + -- 2.39.5