import com.google.common.collect.Lists;
import org.sonar.api.ServerComponent;
import org.sonar.api.platform.ComponentContainer;
+import org.sonar.server.source.IndexSourceLinesStep;
import java.util.List;
steps.add(pico.getComponentByType(DataCleanerStep.class));
// project only
steps.add(pico.getComponentByType(IndexProjectIssuesStep.class));
+ // project only
+ steps.add(pico.getComponentByType(IndexSourceLinesStep.class));
return ImmutableList.copyOf(steps);
}
bulkRequest.request().add(request);
if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) {
executeBulk(bulkRequest);
+ bulkRequest = client.prepareBulk();
}
}
import org.sonar.server.rule.index.RuleNormalizer;
import org.sonar.server.rule.ws.*;
import org.sonar.server.search.*;
-import org.sonar.server.source.CodeColorizers;
-import org.sonar.server.source.DeprecatedSourceDecorator;
-import org.sonar.server.source.HtmlSourceDecorator;
-import org.sonar.server.source.SourceService;
+import org.sonar.server.source.*;
import org.sonar.server.source.index.SourceLineIndexDefinition;
import org.sonar.server.source.index.SourceLineIndexer;
import org.sonar.server.source.ws.*;
pico.addSingleton(ScmAction.class);
pico.addSingleton(SourceLineIndexDefinition.class);
pico.addSingleton(SourceLineIndexer.class);
+ pico.addSingleton(IndexSourceLinesStep.class);
// Duplications
pico.addSingleton(DuplicationsParser.class);
import org.sonar.server.issue.index.IssueNormalizer;
import org.sonar.server.qualityprofile.index.ActiveRuleIndex;
import org.sonar.server.rule.index.RuleIndex;
+import org.sonar.server.source.index.SourceLineIndexer;
import java.util.Date;
import java.util.List;
private final DbClient db;
private final IndexClient index;
+ private final SourceLineIndexer sourceLineIndexer;
- public IndexSynchronizer(DbClient db, IndexClient index) {
+ public IndexSynchronizer(DbClient db, IndexClient index, SourceLineIndexer sourceLineIndexer) {
this.db = db;
this.index = index;
+ this.sourceLineIndexer = sourceLineIndexer;
}
public void execute() {
IssueAuthorizationNormalizer.IssueAuthorizationField.PROJECT.field(), projectUuids);
synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class));
synchronize(session, db.activityDao(), index.get(ActivityIndex.class));
+
+ LOG.info("Indexing of sourceLine records");
+ sourceLineIndexer.indexSourceLines(true);
+
session.commit();
LOG.info("Synchronization done in {}ms...", System.currentTimeMillis() - start);
} finally {
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.source;
+
+import org.sonar.core.component.ComponentDto;
+import org.sonar.core.computation.db.AnalysisReportDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.server.computation.ComputationStep;
+import org.sonar.server.source.index.SourceLineIndexer;
+
+public class IndexSourceLinesStep implements ComputationStep {
+
+ private SourceLineIndexer indexer;
+
+ public IndexSourceLinesStep(SourceLineIndexer indexer) {
+ this.indexer = indexer;
+ }
+
+ @Override
+ public void execute(DbSession session, AnalysisReportDto analysisReportDto, ComponentDto project) {
+ indexer.indexSourceLines(false);
+ }
+
+ @Override
+ public String getDescription() {
+ return "Put source code into search index";
+ }
+
+}
}
public void setHighlighting(String fileUuid) {
- setField(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid);
+ setField(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, fileUuid);
}
public String source() {
public static final String FIELD_PROJECT_UUID = "projectUuid";
public static final String FIELD_FILE_UUID = "fileUuid";
public static final String FIELD_LINE = "line";
- public static final String FIELD_SCM_REVISION = "scm_revision";
- public static final String FIELD_SCM_AUTHOR = "scm_author";
- public static final String FIELD_SCM_DATE = "scm_date";
+ public static final String FIELD_SCM_REVISION = "scmRevision";
+ public static final String FIELD_SCM_AUTHOR = "scmAuthor";
+ public static final String FIELD_SCM_DATE = "scmDate";
public static final String FIELD_HIGHLIGHTING = "highlighting";
public static final String FIELD_SOURCE = "source";
import org.sonar.server.db.DbClient;
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.EsClient;
-import org.sonar.server.es.IssueIndexDefinition;
import java.sql.Connection;
import java.util.Collection;
private UpdateRequest newUpsertRequest(SourceLineDoc lineDoc) {
String projectUuid = lineDoc.projectUuid();
- return new UpdateRequest(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE, lineDoc.key())
+ return new UpdateRequest(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE, lineDoc.key())
.routing(projectUuid)
.doc(lineDoc.getFields())
.upsert(lineDoc.getFields());
import org.sonar.server.db.ResultSetIterator;
import org.sonar.server.db.migrations.SqlUtil;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.io.Reader;
import java.sql.*;
import java.util.Collection;
private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from file_sources";
- private static final String SQL_AFTER_DATE = SQL_ALL + " where i.updated_at>=?";
+ private static final String SQL_AFTER_DATE = SQL_ALL + " where updated_at>=?";
static SourceLineResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) {
try {
String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL;
PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql);
if (afterDate > 0L) {
- stmt.setTimestamp(0, new Timestamp(afterDate));
+ stmt.setTimestamp(1, new Timestamp(afterDate));
}
return new SourceLineResultSetIterator(stmt);
} catch (SQLException e) {
- throw new IllegalStateException("Fail to prepare SQL request to select all issues", e);
+ throw new IllegalStateException("Fail to prepare SQL request to select all file sources", e);
}
}
@Override
protected Collection<SourceLineDoc> read(ResultSet rs) throws SQLException {
-
String projectUuid = rs.getString(1);
String fileUuid = rs.getString(2);
- // createdAt = rs.getDate(3);
Date updatedAt = SqlUtil.getDate(rs, 4);
- Reader dataStream = rs.getClob(5).getCharacterStream();
- // String dataHash = rs.getString(6);
+ Reader dataStream = new InputStreamReader(new ByteArrayInputStream(rs.getBytes(5)));
int line = 1;
List<SourceLineDoc> lines = Lists.newArrayList();
try {
csvParser = new CSVParser(dataStream, CSVFormat.DEFAULT);
- for(CSVRecord record: csvParser) {
- SourceLineDoc doc = new SourceLineDoc(Maps.<String, Object>newHashMapWithExpectedSize(8));
+ for(CSVRecord csvRecord: csvParser) {
+ SourceLineDoc doc = new SourceLineDoc(Maps.<String, Object>newHashMapWithExpectedSize(9));
doc.setProjectUuid(projectUuid);
doc.setFileUuid(fileUuid);
- doc.setLine(line ++);
+ doc.setLine(line);
doc.setUpdateDate(updatedAt);
- doc.setScmRevision(record.get(0));
- doc.setScmAuthor(record.get(1));
- doc.setScmDate(DateUtils.parseDateTimeQuietly(record.get(2)));
- doc.setHighlighting(record.get(3));
- doc.setSource(record.get(4));
+ doc.setScmRevision(csvRecord.get(0));
+ doc.setScmAuthor(csvRecord.get(1));
+ doc.setScmDate(DateUtils.parseDateTimeQuietly(csvRecord.get(2)));
+ doc.setHighlighting(csvRecord.get(3));
+ doc.setSource(csvRecord.get(4));
+
+ lines.add(doc);
+
+ line ++;
}
} catch(IOException ioError) {
+ throw new IllegalStateException("Impossible to open stream for file_sources.data with file_uuid " + fileUuid);
+ } catch(ArrayIndexOutOfBoundsException lineError) {
throw new IllegalStateException(
- String.format("Impossible to parse source line data, stuck at line %d", line), ioError);
+ String.format("Impossible to parse source line data, stuck at line %d", line), lineError);
} finally {
IOUtils.closeQuietly(csvParser);
IOUtils.closeQuietly(dataStream);
import org.junit.ClassRule;
import org.junit.Test;
import org.sonar.api.platform.ComponentContainer;
+import org.sonar.server.source.IndexSourceLinesStep;
import org.sonar.server.tester.ServerTester;
import java.util.List;
pico.addSingleton(mock(DataCleanerStep.class));
pico.addSingleton(mock(InvalidatePreviewCacheStep.class));
pico.addSingleton(mock(ComponentIndexationInDatabaseStep.class));
+ pico.addSingleton(mock(IndexSourceLinesStep.class));
sut = new ComputationStepRegistry(pico);
}
InvalidatePreviewCacheStep.class,
ComponentIndexationInDatabaseStep.class,
DataCleanerStep.class,
- IndexProjectIssuesStep.class
+ IndexProjectIssuesStep.class,
+ IndexSourceLinesStep.class
);
List<ComputationStep> steps = sut.steps();
import org.sonar.server.rule.RuleTesting;
import org.sonar.server.rule.db.RuleDao;
import org.sonar.server.rule.index.RuleIndex;
+import org.sonar.server.source.index.SourceLineIndexer;
import org.sonar.server.tester.ServerTester;
import static org.fest.assertions.Assertions.assertThat;
IndexSynchronizer synchronizer;
DbClient dbClient;
IndexClient indexClient;
+ SourceLineIndexer sourceLineIndexer;
DbSession dbSession;
@Before
public void setUp() throws Exception {
dbClient = tester.get(DbClient.class);
indexClient = tester.get(IndexClient.class);
+ sourceLineIndexer = tester.get(SourceLineIndexer.class);
dbSession = dbClient.openSession(false);
- synchronizer = new IndexSynchronizer(dbClient, indexClient);
+ synchronizer = new IndexSynchronizer(dbClient, indexClient, sourceLineIndexer);
tester.clearDbAndIndexes();
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.source.index;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.config.Settings;
+import org.sonar.server.db.DbClient;
+import org.sonar.server.es.BulkIndexer;
+import org.sonar.server.es.EsTester;
+import org.sonar.server.search.BaseNormalizer;
+import org.sonar.server.source.index.SourceLineDoc;
+import org.sonar.server.source.index.SourceLineIndexDefinition;
+import org.sonar.server.source.index.SourceLineIndexer;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+
+public class SourceLineIndexerTest {
+
+ @Rule
+ public EsTester es = new EsTester().addDefinitions(new SourceLineIndexDefinition(new Settings()));
+
+ private SourceLineIndexer indexer;
+
+ @Before
+ public void setUp() {
+ indexer = new SourceLineIndexer(mock(DbClient.class), es.client());
+ }
+
+ @Test
+ public void should_index_source_lines() {
+ BulkIndexer bulk = new BulkIndexer(es.client(), SourceLineIndexDefinition.INDEX_SOURCE_LINES);
+
+ SourceLineDoc line1 = new SourceLineDoc(ImmutableMap.<String, Object>builder()
+ .put(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "abcd")
+ .put(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh")
+ .put(SourceLineIndexDefinition.FIELD_LINE, 42)
+ .put(SourceLineIndexDefinition.FIELD_SCM_REVISION, "cafebabe")
+ .put(SourceLineIndexDefinition.FIELD_SCM_DATE, "2014-01-01T12:34:56.7+0100")
+ .put(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "polop")
+ .put(SourceLineIndexDefinition.FIELD_SOURCE, "package org.sonar.server.source;")
+ .put(BaseNormalizer.UPDATED_AT_FIELD, new Date())
+ .build());
+ Collection<SourceLineDoc> sourceLines = ImmutableList.of(line1);
+
+ List<Collection<SourceLineDoc>> sourceLineContainer = Lists.newArrayList();
+ sourceLineContainer.add(sourceLines);
+ indexer.indexSourceLines(bulk, sourceLineContainer.iterator());
+ }
+}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.source.index;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.Charsets;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.sonar.api.utils.DateUtils;
+import org.sonar.core.persistence.TestDatabase;
+import org.sonar.server.db.DbClient;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.util.List;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class SourceLineResultSetIteratorTest {
+
+ @ClassRule
+ public static TestDatabase db = new TestDatabase();
+
+ DbClient dbClient;
+
+ Connection connection;
+
+ @Before
+ public void setUp() throws Exception {
+ dbClient = new DbClient(db.database(), db.myBatis());
+ db.schema(this.getClass(), "schema.sql");
+ connection = db.openConnection();
+ }
+
+ @After
+ public void after() throws Exception {
+ connection.close();
+ }
+
+ @Test
+ public void should_generate_source_line_documents() throws Exception {
+ db.prepareDbUnit(getClass(), "source-with-scm.xml");
+ Connection connection = db.openConnection();
+ PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET data = ? WHERE id=1");
+ stmt.setBytes(1, ("aef12a,alice,2014-04-25T12:34:56+0100,,class Foo {\r\n" +
+ "abe465,bob,2014-07-25T12:34:56+0100,, // Empty\r\n" +
+ "afb789,carol,2014-03-23T12:34:56+0100,,}\r\n" +
+ "afb789,carol,2014-03-23T12:34:56+0100,,\r\n").getBytes(Charsets.UTF_8));
+ stmt.executeUpdate();
+ connection.commit();
+
+ SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L);
+ assertThat(iterator.hasNext()).isTrue();
+ List<SourceLineDoc> sourceLines = Lists.newArrayList(iterator.next());
+ assertThat(sourceLines).hasSize(4);
+ SourceLineDoc firstLine = sourceLines.get(0);
+ assertThat(firstLine.projectUuid()).isEqualTo("uuid-MyProject");
+ assertThat(firstLine.fileUuid()).isEqualTo("uuid-MyFile.xoo");
+ assertThat(firstLine.line()).isEqualTo(1);
+ assertThat(firstLine.scmRevision()).isEqualTo("aef12a");
+ assertThat(firstLine.scmAuthor()).isEqualTo("alice");
+ assertThat(firstLine.scmDate()).isEqualTo(DateUtils.parseDateTime("2014-04-25T12:34:56+0100"));
+ assertThat(firstLine.highlighting()).isEmpty();
+ assertThat(firstLine.source()).isEqualTo("class Foo {");
+ }
+
+ @Test
+ public void should_ignore_lines_already_handled() throws Exception {
+ db.prepareDbUnit(getClass(), "source-with-scm.xml");
+
+ SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, db.openConnection(),
+ DateUtils.parseDateTime("2014-11-01T16:44:02+0100").getTime());
+ assertThat(iterator.hasNext()).isFalse();
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void should_fail_on_bad_csv() throws Exception {
+ db.prepareDbUnit(getClass(), "source-with-scm.xml");
+ Connection connection = db.openConnection();
+ PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET data = ? WHERE id=1");
+ stmt.setBytes(1, ("plouf").getBytes(Charsets.UTF_8));
+ stmt.executeUpdate();
+ connection.commit();
+
+ SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L);
+ assertThat(iterator.hasNext()).isTrue();
+ iterator.next();
+ }
+
+}
--- /dev/null
+
+CREATE TABLE "FILE_SOURCES" (
+ "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1),
+ "PROJECT_UUID" VARCHAR(50) NOT NULL,
+ "FILE_UUID" VARCHAR(50) NOT NULL,
+ "DATA" CLOB(2147483647),
+ "DATA_HASH" VARCHAR(50) NOT NULL,
+ "CREATED_AT" TIMESTAMP NOT NULL,
+ "UPDATED_AT" TIMESTAMP NOT NULL
+);
--- /dev/null
+<dataset>
+
+ <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="2014-11-17 16:27:00.000" updated_at="2014-10-31 16:44:02.000"
+ data="" data_hash="" />
+
+</dataset>
projects / sonarqube.git / commitdiff