From: Teryk Bellahsene Date: Thu, 30 Apr 2015 14:24:13 +0000 (+0200) Subject: index tests - SONAR-6255 X-Git-Tag: 5.2-RC1~2029 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=885033d25e863905348f37a566b8cdfc840702b7;p=sonarqube.git index tests - SONAR-6255 --- diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java index 85726d70af8..953f036f12e 100644 --- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java +++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java @@ -31,7 +31,8 @@ import org.sonar.core.source.db.FileSourceDto; import org.sonar.server.db.DbClient; import org.sonar.server.source.db.FileSourceDao; import org.sonar.server.source.db.FileSourceDb; -import org.sonar.server.source.index.SourceFileResultSetIterator; +import org.sonar.server.source.index.FileSourcesUpdaterUtil; +import org.sonar.server.source.index.SourceLineResultSetIterator; import java.io.IOException; import java.sql.Connection; @@ -73,10 +74,10 @@ public class SourceDbBenchmarkTest { try { long start = System.currentTimeMillis(); - SourceFileResultSetIterator it = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); + SourceLineResultSetIterator it = SourceLineResultSetIterator.create(dbClient, connection, 0L, null); while (it.hasNext()) { - SourceFileResultSetIterator.Row row = it.next(); - assertThat(row.getLineUpdateRequests().size()).isEqualTo(NUMBER_OF_LINES); + FileSourcesUpdaterUtil.Row row = it.next(); + assertThat(row.getUpdateRequests().size()).isEqualTo(NUMBER_OF_LINES); assertThat(row.getFileUuid()).isNotEmpty(); counter.incrementAndGet(); } diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java index 23db5b1f112..de4c7e274e8 100644 --- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java +++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java @@ -102,7 +102,7 @@ public class SourceIndexBenchmarkTest { // TODO assertions } - private static class SourceIterator implements Iterator { + private static class SourceIterator implements Iterator { private final long nbFiles; private final int nbLinesPerFile; private int currentProject = 0; @@ -125,7 +125,7 @@ public class SourceIndexBenchmarkTest { } @Override - public SourceFileResultSetIterator.Row next() { + public FileSourcesUpdaterUtil.Row next() { String projectUuid = "P" + currentProject; String fileUuid = "FILE" + count.get(); dataBuilder.clear(); @@ -155,7 +155,7 @@ public class SourceIndexBenchmarkTest { if (count.get() % 500 == 0) { currentProject++; } - return SourceFileResultSetIterator.toRow(projectUuid, fileUuid, new Date(), dataBuilder.build()); + return SourceLineResultSetIterator.toRow(projectUuid, fileUuid, new Date(), dataBuilder.build()); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexTestsStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexTestsStep.java new file mode 100644 index 00000000000..7c44fd7c9b7 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexTestsStep.java @@ -0,0 +1,50 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.step; + +import org.sonar.api.resources.Qualifiers; +import org.sonar.server.computation.ComputationContext; +import org.sonar.server.test.index.TestIndexer; + +public class IndexTestsStep implements ComputationStep { + + private final TestIndexer indexer; + + public IndexTestsStep(TestIndexer indexer) { + this.indexer = indexer; + } + + @Override + public String[] supportedProjectQualifiers() { + return new String[] {Qualifiers.PROJECT}; + } + + @Override + public void execute(ComputationContext context) { + indexer.index(context.getProject().uuid()); + } + + @Override + public String getDescription() { + return "Index tests"; + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/FileSourcesUpdaterUtil.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/FileSourcesUpdaterUtil.java new file mode 100644 index 00000000000..392792e81b3 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/FileSourcesUpdaterUtil.java @@ -0,0 +1,107 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.source.index; + +import com.google.common.base.Joiner; +import org.elasticsearch.action.update.UpdateRequest; +import org.sonar.server.db.DbClient; + +import javax.annotation.Nullable; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class FileSourcesUpdaterUtil { + + private static final String SQL_ALL = "SELECT %s FROM file_sources WHERE data_type='%s' "; + private static final String AFTER_DATE_FILTER = " AND updated_at>?"; + private static final String PROJECT_FILTER = " AND project_uuid=?"; + + private static final String[] FIELDS = { + "project_uuid", + "file_uuid", + "updated_at", + "binary_data" + }; + + private FileSourcesUpdaterUtil() { + // only static stuff + } + + public static PreparedStatement preparedStatementToSelectFileSources(DbClient dbClient, Connection connection, String dataType, long afterDate, @Nullable String projectUuid) throws SQLException { + String sql = createSQL(dataType, afterDate, projectUuid); + // rows are big, so they are scrolled once at a time (one row in memory at a time) + PreparedStatement stmt = dbClient.newScrollingSingleRowSelectStatement(connection, sql); + int index = 1; + if (afterDate > 0L) { + stmt.setLong(index, afterDate); + index++; + } + if (projectUuid != null) { + stmt.setString(index, projectUuid); + } + return stmt; + } + + private static String createSQL(String dataType, long afterDate, @Nullable String projectUuid) { + String sql = String.format(SQL_ALL, Joiner.on(",").join(FIELDS), dataType); + if (afterDate > 0L || projectUuid != null) { + if (afterDate > 0L) { + sql += AFTER_DATE_FILTER; + } + if (projectUuid != null) { + sql += PROJECT_FILTER; + } + } + return sql; + } + + public static class Row { + private final String fileUuid, projectUuid; + private final long updatedAt; + private final List updateRequests = new ArrayList<>(); + + public Row(String projectUuid, String fileUuid, long updatedAt) { + this.projectUuid = projectUuid; + this.fileUuid = fileUuid; + this.updatedAt = updatedAt; + } + + public String getProjectUuid() { + return projectUuid; + } + + public String getFileUuid() { + return fileUuid; + } + + public long getUpdatedAt() { + return updatedAt; + } + + public List getUpdateRequests() { + return updateRequests; + } + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java deleted file mode 100644 index 6c14cd997d1..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.source.index; - -import org.apache.commons.io.Charsets; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.update.UpdateRequest; -import org.sonar.api.utils.text.JsonWriter; -import org.sonar.core.source.db.FileSourceDto; -import org.sonar.core.source.db.FileSourceDto.Type; -import org.sonar.server.db.DbClient; -import org.sonar.server.db.ResultSetIterator; -import org.sonar.server.es.EsUtils; -import org.sonar.server.source.db.FileSourceDb; - -import javax.annotation.Nullable; - -import java.io.ByteArrayOutputStream; -import java.io.OutputStreamWriter; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -/** - * Scroll over table FILE_SOURCES and directly parse data required to - * populate the index sourcelines - */ -public class SourceFileResultSetIterator extends ResultSetIterator { - - public static class Row { - private final String fileUuid, projectUuid; - private final long updatedAt; - private final List lineUpdateRequests = new ArrayList<>(); - - public Row(String projectUuid, String fileUuid, long updatedAt) { - this.projectUuid = projectUuid; - this.fileUuid = fileUuid; - this.updatedAt = updatedAt; - } - - public String getProjectUuid() { - return projectUuid; - } - - public String getFileUuid() { - return fileUuid; - } - - public long getUpdatedAt() { - return updatedAt; - } - - public List getLineUpdateRequests() { - return lineUpdateRequests; - } - } - - private static final String[] FIELDS = { - "project_uuid", - "file_uuid", - "updated_at", - "binary_data" - }; - - private static final String SQL_ALL = String.format("SELECT %s FROM file_sources WHERE data_type='%s' ", StringUtils.join(FIELDS, ","), Type.SOURCE); - private static final String AFTER_DATE_FILTER = " AND updated_at>?"; - private static final String PROJECT_FILTER = " AND project_uuid=?"; - - public static SourceFileResultSetIterator create(DbClient dbClient, Connection connection, long afterDate, @Nullable String projectUuid) { - try { - String sql = createSQL(afterDate, projectUuid); - // rows are big, so they are scrolled once at a time (one row in memory at a time) - PreparedStatement stmt = dbClient.newScrollingSingleRowSelectStatement(connection, sql); - int index = 1; - if (afterDate > 0L) { - stmt.setLong(index, afterDate); - index++; - } - if (projectUuid != null) { - stmt.setString(index, projectUuid); - } - return new SourceFileResultSetIterator(stmt); - } catch (SQLException e) { - throw new IllegalStateException("Fail to prepare SQL request to select all file sources", e); - } - } - - private static String createSQL(long afterDate, @Nullable String projectUuid) { - String sql = SQL_ALL; - if (afterDate > 0L || projectUuid != null) { - if (afterDate > 0L) { - sql += AFTER_DATE_FILTER; - } - if (projectUuid != null) { - sql += PROJECT_FILTER; - } - } - return sql; - } - - private SourceFileResultSetIterator(PreparedStatement stmt) throws SQLException { - super(stmt); - } - - @Override - protected Row read(ResultSet rs) throws SQLException { - String projectUuid = rs.getString(1); - String fileUuid = rs.getString(2); - Date updatedAt = new Date(rs.getLong(3)); - FileSourceDb.Data data = FileSourceDto.decodeSourceData(rs.getBinaryStream(4)); - return toRow(projectUuid, fileUuid, updatedAt, data); - } - - /** - * Convert protobuf message to data required for Elasticsearch indexing - */ - public static Row toRow(String projectUuid, String fileUuid, Date updatedAt, FileSourceDb.Data data) { - Row result = new Row(projectUuid, fileUuid, updatedAt.getTime()); - for (FileSourceDb.Line line : data.getLinesList()) { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - // all the fields must be present, even if value is null - JsonWriter writer = JsonWriter.of(new OutputStreamWriter(bytes, Charsets.UTF_8)).setSerializeNulls(true); - writer.beginObject(); - writer.prop(SourceLineIndexDefinition.FIELD_PROJECT_UUID, projectUuid); - writer.prop(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid); - writer.prop(SourceLineIndexDefinition.FIELD_LINE, line.getLine()); - writer.prop(SourceLineIndexDefinition.FIELD_UPDATED_AT, EsUtils.formatDateTime(updatedAt)); - writer.prop(SourceLineIndexDefinition.FIELD_SCM_REVISION, line.getScmRevision()); - writer.prop(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, line.getScmAuthor()); - writer.prop(SourceLineIndexDefinition.FIELD_SCM_DATE, EsUtils.formatDateTime(line.hasScmDate() ? new Date(line.getScmDate()) : null)); - - // unit tests - if (line.hasUtLineHits()) { - writer.prop(SourceLineIndexDefinition.FIELD_UT_LINE_HITS, line.getUtLineHits()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_UT_LINE_HITS).valueObject(null); - } - if (line.hasUtConditions()) { - writer.prop(SourceLineIndexDefinition.FIELD_UT_CONDITIONS, line.getUtConditions()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_UT_CONDITIONS).valueObject(null); - } - if (line.hasUtCoveredConditions()) { - writer.prop(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, line.getUtCoveredConditions()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS).valueObject(null); - } - - // IT - if (line.hasItLineHits()) { - writer.prop(SourceLineIndexDefinition.FIELD_IT_LINE_HITS, line.getItLineHits()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_IT_LINE_HITS).valueObject(null); - } - if (line.hasItConditions()) { - writer.prop(SourceLineIndexDefinition.FIELD_IT_CONDITIONS, line.getItConditions()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_IT_CONDITIONS).valueObject(null); - } - if (line.hasItCoveredConditions()) { - writer.prop(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, line.getItCoveredConditions()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS).valueObject(null); - } - - // Overall coverage - if (line.hasOverallLineHits()) { - writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, line.getOverallLineHits()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS).valueObject(null); - } - if (line.hasOverallConditions()) { - writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, line.getOverallConditions()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS).valueObject(null); - } - if (line.hasOverallCoveredConditions()) { - writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS, line.getOverallCoveredConditions()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS).valueObject(null); - } - - if (line.hasHighlighting()) { - writer.prop(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, line.getHighlighting()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_HIGHLIGHTING).valueObject(null); - } - if (line.hasSymbols()) { - writer.prop(SourceLineIndexDefinition.FIELD_SYMBOLS, line.getSymbols()); - } else { - writer.name(SourceLineIndexDefinition.FIELD_SYMBOLS).valueObject(null); - } - writer.name(SourceLineIndexDefinition.FIELD_DUPLICATIONS).valueObject(line.getDuplicationList()); - writer.prop(SourceLineIndexDefinition.FIELD_SOURCE, line.hasSource() ? line.getSource() : null); - writer.endObject().close(); - - // This is an optimization to reduce memory consumption and multiple conversions from Map to JSON. - // UpdateRequest#doc() and #upsert() take the same parameter values, so: - // - passing the same Map would execute two JSON serializations - // - Map is a useless temporarily structure: read JDBC result set -> convert to map -> convert to JSON. Generating - // directly JSON from result set is more efficient. - byte[] jsonDoc = bytes.toByteArray(); - UpdateRequest updateRequest = new UpdateRequest(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE, SourceLineIndexDefinition.docKey(fileUuid, line.getLine())) - .routing(projectUuid) - .doc(jsonDoc) - .upsert(jsonDoc); - result.lineUpdateRequests.add(updateRequest); - } - return result; - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java index 8382856261b..fa4b10807a3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java @@ -71,7 +71,7 @@ public class SourceLineIndexer extends BaseIndexer { DbSession dbSession = dbClient.openSession(false); Connection dbConnection = dbSession.getConnection(); try { - SourceFileResultSetIterator rowIt = SourceFileResultSetIterator.create(dbClient, dbConnection, lastUpdatedAt, projectUuid); + SourceLineResultSetIterator rowIt = SourceLineResultSetIterator.create(dbClient, dbConnection, lastUpdatedAt, projectUuid); long maxUpdatedAt = doIndex(bulk, rowIt); rowIt.close(); return maxUpdatedAt; @@ -81,18 +81,18 @@ public class SourceLineIndexer extends BaseIndexer { } } - public long index(Iterator dbRows) { + public long index(Iterator dbRows) { BulkIndexer bulk = new BulkIndexer(esClient, SourceLineIndexDefinition.INDEX); return doIndex(bulk, dbRows); } - private long doIndex(BulkIndexer bulk, Iterator dbRows) { + private long doIndex(BulkIndexer bulk, Iterator dbRows) { long maxUpdatedAt = 0L; bulk.start(); while (dbRows.hasNext()) { - SourceFileResultSetIterator.Row row = dbRows.next(); + FileSourcesUpdaterUtil.Row row = dbRows.next(); addDeleteRequestsForLinesGreaterThan(bulk, row); - for (UpdateRequest updateRequest : row.getLineUpdateRequests()) { + for (UpdateRequest updateRequest : row.getUpdateRequests()) { bulk.add(updateRequest); } maxUpdatedAt = Math.max(maxUpdatedAt, row.getUpdatedAt()); @@ -107,8 +107,8 @@ public class SourceLineIndexer extends BaseIndexer { * - same file has now 5 lines * Lines 6 to 10 must be removed from index. */ - private void addDeleteRequestsForLinesGreaterThan(BulkIndexer bulk, SourceFileResultSetIterator.Row fileRow) { - int numberOfLines = fileRow.getLineUpdateRequests().size(); + private void addDeleteRequestsForLinesGreaterThan(BulkIndexer bulk, FileSourcesUpdaterUtil.Row fileRow) { + int numberOfLines = fileRow.getUpdateRequests().size(); SearchRequestBuilder searchRequest = esClient.prepareSearch(SourceLineIndexDefinition.INDEX) .setTypes(SourceLineIndexDefinition.TYPE) .setRouting(fileRow.getProjectUuid()) diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java new file mode 100644 index 00000000000..e187700d869 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java @@ -0,0 +1,169 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.source.index; + +import org.apache.commons.io.Charsets; +import org.elasticsearch.action.update.UpdateRequest; +import org.sonar.api.utils.text.JsonWriter; +import org.sonar.core.source.db.FileSourceDto; +import org.sonar.server.db.DbClient; +import org.sonar.server.db.ResultSetIterator; +import org.sonar.server.es.EsUtils; +import org.sonar.server.source.db.FileSourceDb; + +import javax.annotation.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; + +import static org.sonar.server.source.index.FileSourcesUpdaterUtil.Row; + +/** + * Scroll over table FILE_SOURCES and directly parse data required to + * populate the index sourcelines + */ +public class SourceLineResultSetIterator extends ResultSetIterator { + + public static SourceLineResultSetIterator create(DbClient dbClient, Connection connection, long afterDate, @Nullable String projectUuid) { + try { + return new SourceLineResultSetIterator(FileSourcesUpdaterUtil.preparedStatementToSelectFileSources(dbClient, connection, FileSourceDto.Type.SOURCE, afterDate, + projectUuid)); + } catch (SQLException e) { + throw new IllegalStateException("Fail to prepare SQL request to select all file sources", e); + } + } + + private SourceLineResultSetIterator(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + @Override + protected Row read(ResultSet rs) throws SQLException { + String projectUuid = rs.getString(1); + String fileUuid = rs.getString(2); + Date updatedAt = new Date(rs.getLong(3)); + FileSourceDb.Data data = FileSourceDto.decodeSourceData(rs.getBinaryStream(4)); + return toRow(projectUuid, fileUuid, updatedAt, data); + } + + /** + * Convert protobuf message to data required for Elasticsearch indexing + */ + public static Row toRow(String projectUuid, String fileUuid, Date updatedAt, FileSourceDb.Data data) { + Row result = new Row(projectUuid, fileUuid, updatedAt.getTime()); + for (FileSourceDb.Line line : data.getLinesList()) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + // all the fields must be present, even if value is null + JsonWriter writer = JsonWriter.of(new OutputStreamWriter(bytes, Charsets.UTF_8)).setSerializeNulls(true); + writer.beginObject(); + writer.prop(SourceLineIndexDefinition.FIELD_PROJECT_UUID, projectUuid); + writer.prop(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid); + writer.prop(SourceLineIndexDefinition.FIELD_LINE, line.getLine()); + writer.prop(SourceLineIndexDefinition.FIELD_UPDATED_AT, EsUtils.formatDateTime(updatedAt)); + writer.prop(SourceLineIndexDefinition.FIELD_SCM_REVISION, line.getScmRevision()); + writer.prop(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, line.getScmAuthor()); + writer.prop(SourceLineIndexDefinition.FIELD_SCM_DATE, EsUtils.formatDateTime(line.hasScmDate() ? new Date(line.getScmDate()) : null)); + + // unit tests + if (line.hasUtLineHits()) { + writer.prop(SourceLineIndexDefinition.FIELD_UT_LINE_HITS, line.getUtLineHits()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_UT_LINE_HITS).valueObject(null); + } + if (line.hasUtConditions()) { + writer.prop(SourceLineIndexDefinition.FIELD_UT_CONDITIONS, line.getUtConditions()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_UT_CONDITIONS).valueObject(null); + } + if (line.hasUtCoveredConditions()) { + writer.prop(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, line.getUtCoveredConditions()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS).valueObject(null); + } + + // IT + if (line.hasItLineHits()) { + writer.prop(SourceLineIndexDefinition.FIELD_IT_LINE_HITS, line.getItLineHits()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_IT_LINE_HITS).valueObject(null); + } + if (line.hasItConditions()) { + writer.prop(SourceLineIndexDefinition.FIELD_IT_CONDITIONS, line.getItConditions()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_IT_CONDITIONS).valueObject(null); + } + if (line.hasItCoveredConditions()) { + writer.prop(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, line.getItCoveredConditions()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS).valueObject(null); + } + + // Overall coverage + if (line.hasOverallLineHits()) { + writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, line.getOverallLineHits()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS).valueObject(null); + } + if (line.hasOverallConditions()) { + writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, line.getOverallConditions()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS).valueObject(null); + } + if (line.hasOverallCoveredConditions()) { + writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS, line.getOverallCoveredConditions()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS).valueObject(null); + } + + if (line.hasHighlighting()) { + writer.prop(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, line.getHighlighting()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_HIGHLIGHTING).valueObject(null); + } + if (line.hasSymbols()) { + writer.prop(SourceLineIndexDefinition.FIELD_SYMBOLS, line.getSymbols()); + } else { + writer.name(SourceLineIndexDefinition.FIELD_SYMBOLS).valueObject(null); + } + writer.name(SourceLineIndexDefinition.FIELD_DUPLICATIONS).valueObject(line.getDuplicationList()); + writer.prop(SourceLineIndexDefinition.FIELD_SOURCE, line.hasSource() ? line.getSource() : null); + writer.endObject().close(); + + // This is an optimization to reduce memory consumption and multiple conversions from Map to JSON. + // UpdateRequest#doc() and #upsert() take the same parameter values, so: + // - passing the same Map would execute two JSON serializations + // - Map is a useless temporarily structure: read JDBC result set -> convert to map -> convert to JSON. Generating + // directly JSON from result set is more efficient. + byte[] jsonDoc = bytes.toByteArray(); + UpdateRequest updateRequest = new UpdateRequest(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE, SourceLineIndexDefinition.docKey(fileUuid, line.getLine())) + .routing(projectUuid) + .doc(jsonDoc) + .upsert(jsonDoc); + result.getUpdateRequests().add(updateRequest); + } + return result; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java index a6da0650834..7d0210e9468 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java @@ -36,15 +36,33 @@ public class TestDoc extends BaseDoc { @VisibleForTesting TestDoc() { - super(Maps.newHashMapWithExpectedSize(7)); + super(Maps.newHashMapWithExpectedSize(10)); } - public String uuid() { - return getField(FIELD_UUID); + public String projectUuid() { + return getField(FIELD_PROJECT_UUID); } - public TestDoc setUuid(String uuid) { - setField(FIELD_UUID, uuid); + public TestDoc setProjectUuid(String projectUuid) { + setField(FIELD_PROJECT_UUID, projectUuid); + return this; + } + + public String fileUuid() { + return getField(FIELD_FILE_UUID); + } + + public TestDoc setFileUuid(String fileUuid) { + setField(FIELD_FILE_UUID, fileUuid); + return this; + } + + public String testUuid() { + return getField(FIELD_TEST_UUID); + } + + public TestDoc setUuid(String testUuid) { + setField(FIELD_TEST_UUID, testUuid); return this; } @@ -84,15 +102,6 @@ public class TestDoc extends BaseDoc { return this; } - public String type() { - return getField(FIELD_TYPE); - } - - public TestDoc setType(String type) { - setField(FIELD_TYPE, type); - return this; - } - public Long durationInMs() { return getField(FIELD_DURATION_IN_MS); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java index dce696199ab..af3abccc405 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java @@ -44,7 +44,7 @@ public class TestIndex extends BaseIndex { .setTypes(TestIndexDefinition.TYPE) .setSize(1) .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.boolFilter() - .must(FilterBuilders.termFilter(FIELD_UUID, testFileUuid).cache(false)) + .must(FilterBuilders.termFilter(FIELD_FILE_UUID, testFileUuid).cache(false)) .must(FilterBuilders.termFilter(TestIndexDefinition.FIELD_NAME, methodName).cache(false)))) .get().getHits().getHits()) { coverageBlocks.addAll(new TestDoc(hit.sourceAsMap()).coverageBlocks()); @@ -59,7 +59,7 @@ public class TestIndex extends BaseIndex { for (SearchHit hit : getClient().prepareSearch(TestIndexDefinition.INDEX) .setTypes(TestIndexDefinition.TYPE) .setSize(10_000) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(FIELD_UUID, testFileUuid))) + .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(FIELD_FILE_UUID, testFileUuid))) .get().getHits().getHits()) { testDocs.add(new TestDoc(hit.sourceAsMap())); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java index 8c0b119c92a..7aef0a61fa9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java @@ -27,16 +27,18 @@ import org.sonar.server.es.NewIndex; public class TestIndexDefinition implements IndexDefinition { public static final String INDEX = "tests"; public static final String TYPE = "test"; - public static final String FIELD_UUID = "uuid"; + public static final String FIELD_PROJECT_UUID = "projectUuid"; + public static final String FIELD_FILE_UUID = "fileUuid"; + public static final String FIELD_TEST_UUID = "testUuid"; public static final String FIELD_NAME = "name"; public static final String FIELD_STATUS = "status"; - public static final String FIELD_TYPE = "type"; public static final String FIELD_DURATION_IN_MS = "durationInMs"; public static final String FIELD_MESSAGE = "message"; public static final String FIELD_STACKTRACE = "stacktrace"; public static final String FIELD_COVERAGE_BLOCKS = "coverageBlocks"; public static final String FIELD_COVERAGE_BLOCK_UUID = "uuid"; public static final String FIELD_COVERAGE_BLOCK_LINES = "lines"; + public static final String FIELD_UPDATED_AT = "updatedAt"; private final Settings settings; @@ -56,13 +58,15 @@ public class TestIndexDefinition implements IndexDefinition { nestedMapping.createIntegerField(FIELD_COVERAGE_BLOCK_LINES); NewIndex.NewIndexType mapping = index.createType(TYPE); - mapping.stringFieldBuilder(FIELD_UUID).build(); + mapping.stringFieldBuilder(FIELD_PROJECT_UUID).build(); + mapping.stringFieldBuilder(FIELD_FILE_UUID).build(); + mapping.stringFieldBuilder(FIELD_TEST_UUID).build(); mapping.stringFieldBuilder(FIELD_NAME).build(); mapping.stringFieldBuilder(FIELD_STATUS).disableSearch().build(); - mapping.stringFieldBuilder(FIELD_TYPE).disableSearch().build(); mapping.createLongField(FIELD_DURATION_IN_MS); mapping.stringFieldBuilder(FIELD_MESSAGE).disableSearch().build(); mapping.stringFieldBuilder(FIELD_STACKTRACE).disableSearch().build(); mapping.nestedObjectBuilder(FIELD_COVERAGE_BLOCKS, nestedMapping).build(); + mapping.createDateTimeField(FIELD_UPDATED_AT); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java new file mode 100644 index 00000000000..13da026477d --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java @@ -0,0 +1,123 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.test.index; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilders; +import org.sonar.core.persistence.DbSession; +import org.sonar.server.db.DbClient; +import org.sonar.server.es.BaseIndexer; +import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.EsClient; +import org.sonar.server.source.index.FileSourcesUpdaterUtil; + +import javax.annotation.Nullable; +import java.sql.Connection; +import java.util.Iterator; + +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_PROJECT_UUID; +import static org.sonar.server.test.index.TestIndexDefinition.*; + +/** + * Add to Elasticsearch index {@link TestIndexDefinition} the rows of + * db table FILE_SOURCES that are not indexed yet + */ +public class TestIndexer extends BaseIndexer { + + private final DbClient dbClient; + + public TestIndexer(DbClient dbClient, EsClient esClient) { + super(esClient, 0L, INDEX, TYPE, FIELD_UPDATED_AT); + this.dbClient = dbClient; + } + + public void index(final String projectUuid) { + super.index(new IndexerTask() { + @Override + public long index(long lastUpdatedAt) { + return doIndex(lastUpdatedAt, projectUuid); + } + }); + } + + @Override + protected long doIndex(long lastUpdatedAt) { + return doIndex(lastUpdatedAt, null); + } + + private long doIndex(long lastUpdatedAt, @Nullable String projectUuid) { + final BulkIndexer bulk = new BulkIndexer(esClient, INDEX); + bulk.setLarge(lastUpdatedAt == 0L); + + DbSession dbSession = dbClient.openSession(false); + Connection dbConnection = dbSession.getConnection(); + try { + TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbConnection, lastUpdatedAt, projectUuid); + long maxUpdatedAt = doIndex(bulk, rowIt); + rowIt.close(); + return maxUpdatedAt; + + } finally { + dbSession.close(); + } + } + + public long index(Iterator dbRows) { + BulkIndexer bulk = new BulkIndexer(esClient, INDEX); + return doIndex(bulk, dbRows); + } + + private long doIndex(BulkIndexer bulk, Iterator dbRows) { + long maxUpdatedAt = 0L; + bulk.start(); + while (dbRows.hasNext()) { + FileSourcesUpdaterUtil.Row row = dbRows.next(); + for (UpdateRequest updateRequest : row.getUpdateRequests()) { + bulk.add(updateRequest); + } + maxUpdatedAt = Math.max(maxUpdatedAt, row.getUpdatedAt()); + } + bulk.stop(); + return maxUpdatedAt; + } + + public void deleteByFile(String fileUuid) { + // TODO would be great to have the projectUuid for routing + SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX) + .setTypes(TYPE) + .setQuery(QueryBuilders.filteredQuery( + QueryBuilders.matchAllQuery(), + FilterBuilders.termFilter(FIELD_FILE_UUID, fileUuid).cache(false))); + BulkIndexer.delete(esClient, INDEX, searchRequest); + } + + public void deleteByProject(String projectUuid) { + SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX) + .setRouting(projectUuid) + .setTypes(TYPE) + .setQuery(QueryBuilders.filteredQuery( + QueryBuilders.matchAllQuery(), + FilterBuilders.termFilter(FIELD_PROJECT_UUID, projectUuid).cache(false))); + BulkIndexer.delete(esClient, INDEX, searchRequest); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestResultSetIterator.java new file mode 100644 index 00000000000..e8827abf96a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestResultSetIterator.java @@ -0,0 +1,117 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.test.index; + +import org.apache.commons.io.Charsets; +import org.elasticsearch.action.update.UpdateRequest; +import org.sonar.api.utils.text.JsonWriter; +import org.sonar.core.source.db.FileSourceDto; +import org.sonar.server.db.DbClient; +import org.sonar.server.db.ResultSetIterator; +import org.sonar.server.source.db.FileSourceDb; +import org.sonar.server.source.index.FileSourcesUpdaterUtil; +import org.sonar.server.source.index.FileSourcesUpdaterUtil.Row; + +import javax.annotation.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; +import java.util.List; + +import static org.sonar.server.test.index.TestIndexDefinition.*; + +/** + * Scroll over table FILE_SOURCES and directly parse data required to + * populate the index sourcelines + */ +public class TestResultSetIterator extends ResultSetIterator { + + public static TestResultSetIterator create(DbClient dbClient, Connection connection, long afterDate, @Nullable String projectUuid) { + try { + return new TestResultSetIterator(FileSourcesUpdaterUtil.preparedStatementToSelectFileSources(dbClient, connection, FileSourceDto.Type.TEST, afterDate, projectUuid)); + } catch (SQLException e) { + throw new IllegalStateException("Fail to prepare SQL request to select all tests", e); + } + } + + private TestResultSetIterator(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + @Override + protected Row read(ResultSet rs) throws SQLException { + String projectUuid = rs.getString(1); + String fileUuid = rs.getString(2); + Date updatedAt = new Date(rs.getLong(3)); + List data = FileSourceDto.decodeTestData(rs.getBinaryStream(4)); + return toRow(projectUuid, fileUuid, updatedAt, data); + } + + /** + * Convert protobuf message to tests required for Elasticsearch indexing + */ + public static Row toRow(String projectUuid, String fileUuid, Date updatedAt, List tests) { + Row result = new Row(projectUuid, fileUuid, updatedAt.getTime()); + for (FileSourceDb.Test test : tests) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + // all the fields must be present, even if value is null + JsonWriter writer = JsonWriter.of(new OutputStreamWriter(bytes, Charsets.UTF_8)).setSerializeNulls(true); + writer.beginObject(); + writer.prop(FIELD_PROJECT_UUID, projectUuid); + writer.prop(FIELD_FILE_UUID, fileUuid); + writer.prop(FIELD_TEST_UUID, test.getUuid()); + writer.prop(FIELD_NAME, test.getName()); + writer.prop(FIELD_STATUS, test.hasStatus() ? test.getStatus().toString() : null); + writer.prop(FIELD_DURATION_IN_MS, test.hasExecutionTimeMs() ? test.getExecutionTimeMs() : null); + writer.prop(FIELD_MESSAGE, test.hasMsg() ? test.getMsg() : null); + writer.prop(FIELD_STACKTRACE, test.hasStacktrace() ? test.getStacktrace() : null); + writer.name(FIELD_COVERAGE_BLOCKS); + writer.beginArray(); + for (FileSourceDb.Test.CoveredFile coveredFile : test.getCoveredFileList()) { + writer.beginObject(); + writer.prop(FIELD_COVERAGE_BLOCK_UUID, coveredFile.getFileUuid()); + writer.name(FIELD_COVERAGE_BLOCK_LINES).valueObject(coveredFile.getCoveredLineList()); + writer.endObject(); + } + writer.endArray(); + writer.endObject().close(); + + // This is an optimization to reduce memory consumption and multiple conversions from Map to JSON. + // UpdateRequest#doc() and #upsert() take the same parameter values, so: + // - passing the same Map would execute two JSON serializations + // - Map is a useless temporarily structure: read JDBC result set -> convert to map -> convert to JSON. Generating + // directly JSON from result set is more efficient. + byte[] jsonDoc = bytes.toByteArray(); + UpdateRequest updateRequest = new UpdateRequest(INDEX, TYPE, test.getUuid()) + .routing(projectUuid) + .doc(jsonDoc) + .upsert(jsonDoc); + result.getUpdateRequests().add(updateRequest); + } + return result; + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryLoaderMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryLoaderMediumTest.java index b08ec9817bd..79dfba70c7c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryLoaderMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/batch/ProjectRepositoryLoaderMediumTest.java @@ -20,6 +20,7 @@ package org.sonar.server.batch; +import org.sonar.core.source.db.FileSourceDto.Type; import org.sonar.server.source.db.FileSourceDao; import com.google.common.collect.ImmutableMap; @@ -843,6 +844,7 @@ public class ProjectRepositoryLoaderMediumTest { .setLineHashes("8d7b3d6b83c0a517eac07e1aac94b773") .setCreatedAt(System.currentTimeMillis()) .setUpdatedAt(System.currentTimeMillis()) + .setDataType(Type.SOURCE) .setSrcHash("123456"); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexTestsStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexTestsStepTest.java new file mode 100644 index 00000000000..0dd50aec1e5 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexTestsStepTest.java @@ -0,0 +1,90 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.step; + +import org.elasticsearch.search.SearchHit; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.api.resources.Qualifiers; +import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.core.persistence.DbTester; +import org.sonar.server.component.ComponentTesting; +import org.sonar.server.computation.ComputationContext; +import org.sonar.server.db.DbClient; +import org.sonar.server.es.EsTester; +import org.sonar.server.source.db.FileSourceDao; +import org.sonar.server.test.db.TestTesting; +import org.sonar.server.test.index.TestDoc; +import org.sonar.server.test.index.TestIndexDefinition; +import org.sonar.server.test.index.TestIndexer; + +import java.io.IOException; +import java.sql.Connection; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +public class IndexTestsStepTest extends BaseStepTest { + + @ClassRule + public static DbTester dbTester = new DbTester(); + + @ClassRule + public static EsTester esTester = new EsTester().addDefinitions(new TestIndexDefinition(new Settings())); + + DbClient dbClient; + + @Before + public void setUp() throws Exception { + dbClient = new DbClient(dbTester.database(), dbTester.myBatis(), new FileSourceDao(null)); + } + + @Override + protected ComputationStep step() throws IOException { + TestIndexer testIndexer = new TestIndexer(dbClient, esTester.client()); + testIndexer.setEnabled(true); + return new IndexTestsStep(testIndexer); + } + + @Test + public void supported_project_qualifiers() throws Exception { + assertThat(step().supportedProjectQualifiers()).containsOnly(Qualifiers.PROJECT); + } + + @Test + public void index_source() throws Exception { + dbTester.prepareDbUnit(getClass(), "index_source.xml"); + Connection connection = dbTester.openConnection(); + TestTesting.updateDataColumn(connection, "FILE1_UUID", TestTesting.newRandomTests(1)); + connection.close(); + + step().execute(new ComputationContext(mock(BatchReportReader.class), ComponentTesting.newProjectDto("ABCD"))); + + List docs = esTester.getDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE); + assertThat(docs).hasSize(1); + TestDoc doc = new TestDoc(docs.get(0).sourceAsMap()); + assertThat(doc.projectUuid()).isEqualTo("ABCD"); + assertThat(doc.fileUuid()).isEqualTo("FILE1_UUID"); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index 9bf9a4af552..1ba00b7c1e4 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -59,7 +59,8 @@ import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; import org.sonar.server.rule.db.RuleDao; import org.sonar.server.source.db.FileSourceDb; -import org.sonar.server.source.index.SourceFileResultSetIterator; +import org.sonar.server.source.index.FileSourcesUpdaterUtil; +import org.sonar.server.source.index.SourceLineResultSetIterator; import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.tester.ServerTester; import org.sonar.server.user.MockUserSession; @@ -597,7 +598,7 @@ public class IssueServiceMediumTest { .setLine(line) .setScmAuthor(scmAuthor) .build(); - SourceFileResultSetIterator.Row row = SourceFileResultSetIterator.toRow(file.projectUuid(), file.uuid(), new Date(), dataBuilder.build()); + FileSourcesUpdaterUtil.Row row = SourceLineResultSetIterator.toRow(file.projectUuid(), file.uuid(), new Date(), dataBuilder.build()); tester.get(SourceLineIndexer.class).index(Iterators.singletonIterator(row)); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/db/FileSourceTesting.java b/server/sonar-server/src/test/java/org/sonar/server/source/db/FileSourceTesting.java index 67b8c53c55d..60e5edff68a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/db/FileSourceTesting.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/db/FileSourceTesting.java @@ -40,7 +40,7 @@ public class FileSourceTesting { } public static void updateDataColumn(Connection connection, String fileUuid, byte[] data) throws SQLException { - PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET binary_data = ? WHERE file_uuid=?"); + PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET binary_data = ? WHERE file_uuid=? AND data_type='SOURCE'"); stmt.setBytes(1, data); stmt.setString(2, fileUuid); stmt.executeUpdate(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceFileResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceFileResultSetIteratorTest.java deleted file mode 100644 index 1b6942ffdfc..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceFileResultSetIteratorTest.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.source.index; - -import org.assertj.core.data.MapEntry; -import org.elasticsearch.action.update.UpdateRequest; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.sonar.core.persistence.DbTester; -import org.sonar.server.db.DbClient; -import org.sonar.server.source.db.FileSourceDb; -import org.sonar.server.source.db.FileSourceTesting; -import org.sonar.test.DbTests; - -import java.sql.Connection; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -@Category(DbTests.class) -public class SourceFileResultSetIteratorTest { - - @ClassRule - public static DbTester db = new DbTester().schema(SourceFileResultSetIteratorTest.class, "schema.sql"); - - DbClient dbClient; - - Connection connection; - - SourceFileResultSetIterator iterator; - - @Before - public void setUp() throws Exception { - dbClient = new DbClient(db.database(), db.myBatis()); - connection = db.openConnection(); - } - - @After - public void after() throws Exception { - if (iterator != null) { - iterator.close(); - } - connection.close(); - } - - @Test - public void traverse_db() throws Exception { - db.prepareDbUnit(getClass(), "shared.xml"); - FileSourceTesting.updateDataColumn(connection, "F1", FileSourceTesting.newFakeData(3).build()); - - iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); - assertThat(iterator.hasNext()).isTrue(); - SourceFileResultSetIterator.Row row = iterator.next(); - assertThat(row.getProjectUuid()).isEqualTo("P1"); - assertThat(row.getFileUuid()).isEqualTo("F1"); - assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); - assertThat(row.getLineUpdateRequests()).hasSize(3); - - UpdateRequest firstRequest = row.getLineUpdateRequests().get(0); - Map doc = firstRequest.doc().sourceAsMap(); - assertThat(doc).contains( - MapEntry.entry(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "P1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_FILE_UUID, "F1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_LINE, 1), - MapEntry.entry(SourceLineIndexDefinition.FIELD_SCM_REVISION, "REVISION_1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "AUTHOR_1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, "HIGHLIGHTING_1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_SYMBOLS, "SYMBOLS_1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_UT_LINE_HITS, 1), - MapEntry.entry(SourceLineIndexDefinition.FIELD_UT_CONDITIONS, 2), - MapEntry.entry(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, 3), - MapEntry.entry(SourceLineIndexDefinition.FIELD_IT_LINE_HITS, 4), - MapEntry.entry(SourceLineIndexDefinition.FIELD_IT_CONDITIONS, 5), - MapEntry.entry(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, 6), - MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, 7), - MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, 8), - MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS, 9) - ); - } - - /** - * File with one line. No metadata available on the line. - */ - @Test - public void minimal_data() throws Exception { - db.prepareDbUnit(getClass(), "shared.xml"); - FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); - dataBuilder.addLinesBuilder().setLine(1).build(); - FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); - - iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); - SourceFileResultSetIterator.Row row = iterator.next(); - assertThat(row.getProjectUuid()).isEqualTo("P1"); - assertThat(row.getFileUuid()).isEqualTo("F1"); - assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); - assertThat(row.getLineUpdateRequests()).hasSize(1); - UpdateRequest firstRequest = row.getLineUpdateRequests().get(0); - Map doc = firstRequest.doc().sourceAsMap(); - assertThat(doc).contains( - MapEntry.entry(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "P1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_FILE_UUID, "F1"), - MapEntry.entry(SourceLineIndexDefinition.FIELD_LINE, 1) - ); - // null values - assertThat(doc).containsKeys( - SourceLineIndexDefinition.FIELD_SCM_REVISION, - SourceLineIndexDefinition.FIELD_SCM_AUTHOR, - SourceLineIndexDefinition.FIELD_HIGHLIGHTING, - SourceLineIndexDefinition.FIELD_SYMBOLS, - SourceLineIndexDefinition.FIELD_UT_LINE_HITS, - SourceLineIndexDefinition.FIELD_UT_CONDITIONS, - SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, - SourceLineIndexDefinition.FIELD_IT_LINE_HITS, - SourceLineIndexDefinition.FIELD_IT_CONDITIONS, - SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, - SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, - SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, - SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS - ); - } - - @Test - public void filter_by_date() throws Exception { - db.prepareDbUnit(getClass(), "shared.xml"); - - iterator = SourceFileResultSetIterator.create(dbClient, connection, 2000000000000L, null); - assertThat(iterator.hasNext()).isFalse(); - } - - @Test - public void filter_by_project() throws Exception { - db.prepareDbUnit(getClass(), "filter_by_project.xml"); - FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); - dataBuilder.addLinesBuilder().setLine(1).build(); - FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); - - iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, "P1"); - - SourceFileResultSetIterator.Row row = iterator.next(); - assertThat(row.getProjectUuid()).isEqualTo("P1"); - assertThat(row.getFileUuid()).isEqualTo("F1"); - - // File from other project P2 is not returned - assertThat(iterator.hasNext()).isFalse(); - } - - @Test - public void filter_by_project_and_date() throws Exception { - db.prepareDbUnit(getClass(), "filter_by_project_and_date.xml"); - FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); - dataBuilder.addLinesBuilder().setLine(1).build(); - FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); - - iterator = SourceFileResultSetIterator.create(dbClient, connection, 1400000000000L, "P1"); - - SourceFileResultSetIterator.Row row = iterator.next(); - assertThat(row.getProjectUuid()).isEqualTo("P1"); - assertThat(row.getFileUuid()).isEqualTo("F1"); - - // File F2 is not returned - assertThat(iterator.hasNext()).isFalse(); - } - - @Test - public void fail_on_bad_data_format() throws Exception { - db.prepareDbUnit(getClass(), "shared.xml"); - - FileSourceTesting.updateDataColumn(connection, "F1", "THIS_IS_NOT_PROTOBUF".getBytes()); - - iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); - try { - assertThat(iterator.hasNext()).isTrue(); - iterator.next(); - fail(); - } catch (IllegalStateException e) { - // ok - } - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java index 2a0952307c0..e3f4bf04ae3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java @@ -140,7 +140,7 @@ public class SourceLineIndexerTest { .setSource("new source") .addAllDuplication(duplications) .build(); - SourceFileResultSetIterator.Row dbRow = SourceFileResultSetIterator.toRow("P1", "F1", new Date(), dataBuilder.build()); + FileSourcesUpdaterUtil.Row dbRow = SourceLineResultSetIterator.toRow("P1", "F1", new Date(), dataBuilder.build()); indexer.index(Iterators.singletonIterator(dbRow)); assertThat(countDocuments()).isEqualTo(2L); @@ -214,7 +214,7 @@ public class SourceLineIndexerTest { .setOverallCoveredConditions(bigValue) .build(); - SourceFileResultSetIterator.Row row = SourceFileResultSetIterator.toRow("P1", "F1", new Date(), dataBuilder.build()); + FileSourcesUpdaterUtil.Row row = SourceLineResultSetIterator.toRow("P1", "F1", new Date(), dataBuilder.build()); indexer.index(Iterators.singletonIterator(row)); List hits = getDocuments(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java new file mode 100644 index 00000000000..0429ac3fd18 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java @@ -0,0 +1,200 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.source.index; + +import org.assertj.core.data.MapEntry; +import org.elasticsearch.action.update.UpdateRequest; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.sonar.core.persistence.DbTester; +import org.sonar.server.db.DbClient; +import org.sonar.server.source.db.FileSourceDb; +import org.sonar.server.source.db.FileSourceTesting; +import org.sonar.test.DbTests; + +import java.sql.Connection; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +@Category(DbTests.class) +public class SourceLineResultSetIteratorTest { + + @ClassRule + public static DbTester db = new DbTester().schema(SourceLineResultSetIteratorTest.class, "schema.sql"); + + DbClient dbClient; + + Connection connection; + + SourceLineResultSetIterator iterator; + + @Before + public void setUp() throws Exception { + dbClient = new DbClient(db.database(), db.myBatis()); + connection = db.openConnection(); + } + + @After + public void after() throws Exception { + if (iterator != null) { + iterator.close(); + } + connection.close(); + } + + @Test + public void traverse_db() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + FileSourceTesting.updateDataColumn(connection, "F1", FileSourceTesting.newFakeData(3).build()); + + iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L, null); + assertThat(iterator.hasNext()).isTrue(); + FileSourcesUpdaterUtil.Row row = iterator.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); + assertThat(row.getUpdateRequests()).hasSize(3); + + UpdateRequest firstRequest = row.getUpdateRequests().get(0); + Map doc = firstRequest.doc().sourceAsMap(); + assertThat(doc).contains( + MapEntry.entry(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "P1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_FILE_UUID, "F1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_LINE, 1), + MapEntry.entry(SourceLineIndexDefinition.FIELD_SCM_REVISION, "REVISION_1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "AUTHOR_1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, "HIGHLIGHTING_1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_SYMBOLS, "SYMBOLS_1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_UT_LINE_HITS, 1), + MapEntry.entry(SourceLineIndexDefinition.FIELD_UT_CONDITIONS, 2), + MapEntry.entry(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, 3), + MapEntry.entry(SourceLineIndexDefinition.FIELD_IT_LINE_HITS, 4), + MapEntry.entry(SourceLineIndexDefinition.FIELD_IT_CONDITIONS, 5), + MapEntry.entry(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, 6), + MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, 7), + MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, 8), + MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS, 9) + ); + } + + /** + * File with one line. No metadata available on the line. + */ + @Test + public void minimal_data() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); + dataBuilder.addLinesBuilder().setLine(1).build(); + FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); + + iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L, null); + FileSourcesUpdaterUtil.Row row = iterator.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); + assertThat(row.getUpdateRequests()).hasSize(1); + UpdateRequest firstRequest = row.getUpdateRequests().get(0); + Map doc = firstRequest.doc().sourceAsMap(); + assertThat(doc).contains( + MapEntry.entry(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "P1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_FILE_UUID, "F1"), + MapEntry.entry(SourceLineIndexDefinition.FIELD_LINE, 1) + ); + // null values + assertThat(doc).containsKeys( + SourceLineIndexDefinition.FIELD_SCM_REVISION, + SourceLineIndexDefinition.FIELD_SCM_AUTHOR, + SourceLineIndexDefinition.FIELD_HIGHLIGHTING, + SourceLineIndexDefinition.FIELD_SYMBOLS, + SourceLineIndexDefinition.FIELD_UT_LINE_HITS, + SourceLineIndexDefinition.FIELD_UT_CONDITIONS, + SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, + SourceLineIndexDefinition.FIELD_IT_LINE_HITS, + SourceLineIndexDefinition.FIELD_IT_CONDITIONS, + SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, + SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, + SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, + SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS + ); + } + + @Test + public void filter_by_date() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + + iterator = SourceLineResultSetIterator.create(dbClient, connection, 2000000000000L, null); + assertThat(iterator.hasNext()).isFalse(); + } + + @Test + public void filter_by_project() throws Exception { + db.prepareDbUnit(getClass(), "filter_by_project.xml"); + FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); + dataBuilder.addLinesBuilder().setLine(1).build(); + FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); + + iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L, "P1"); + + FileSourcesUpdaterUtil.Row row = iterator.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + + // File from other project P2 is not returned + assertThat(iterator.hasNext()).isFalse(); + } + + @Test + public void filter_by_project_and_date() throws Exception { + db.prepareDbUnit(getClass(), "filter_by_project_and_date.xml"); + FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); + dataBuilder.addLinesBuilder().setLine(1).build(); + FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); + + iterator = SourceLineResultSetIterator.create(dbClient, connection, 1400000000000L, "P1"); + + FileSourcesUpdaterUtil.Row row = iterator.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + + // File F2 is not returned + assertThat(iterator.hasNext()).isFalse(); + } + + @Test + public void fail_on_bad_data_format() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + + FileSourceTesting.updateDataColumn(connection, "F1", "THIS_IS_NOT_PROTOBUF".getBytes()); + + iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L, null); + try { + assertThat(iterator.hasNext()).isTrue(); + iterator.next(); + fail(); + } catch (IllegalStateException e) { + // ok + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/db/TestTesting.java b/server/sonar-server/src/test/java/org/sonar/server/test/db/TestTesting.java new file mode 100644 index 00000000000..7c95d51d9d7 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/test/db/TestTesting.java @@ -0,0 +1,102 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.test.db; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang.math.RandomUtils; +import org.sonar.api.utils.internal.Uuids; +import org.sonar.batch.protocol.Constants; +import org.sonar.core.source.db.FileSourceDto; +import org.sonar.server.source.db.FileSourceDb; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class TestTesting { + + private TestTesting() { + // only static stuff + } + + public static void updateDataColumn(Connection connection, String fileUuid, List tests) throws SQLException { + updateDataColumn(connection, fileUuid, FileSourceDto.encodeTestData(tests)); + } + + public static void updateDataColumn(Connection connection, String fileUuid, byte[] data) throws SQLException { + PreparedStatement stmt = connection.prepareStatement("UPDATE file_sources SET binary_data = ? WHERE file_uuid=? AND data_type='TEST'"); + stmt.setBytes(1, data); + stmt.setString(2, fileUuid); + stmt.executeUpdate(); + stmt.close(); + } + + public static List newFakeTests(int numberOfTests) throws IOException { + List tests = new ArrayList<>(); + for (int i = 1; i <= numberOfTests; i++) { + FileSourceDb.Test.Builder test = FileSourceDb.Test.newBuilder() + .setUuid("TEST_FILE_UUID_" + i) + .setName("NAME_" + i) + .setStatus(Constants.TestStatus.FAILURE) + .setStacktrace("STACKTRACE_" + i) + .setMsg("MESSAGE_" + i) + .setExecutionTimeMs(i); + for (int j = 1; j <= numberOfTests; j++) { + test.addCoveredFile( + FileSourceDb.Test.CoveredFile.newBuilder() + .setFileUuid("MAIN_FILE_UUID_" + j) + .addCoveredLine(j) + ); + } + tests.add(test.build()); + } + return tests; + } + + /** + * Generate random data. + */ + public static List newRandomTests(int numberOfTests) throws IOException { + List tests = new ArrayList<>(); + for (int i = 1; i <= numberOfTests; i++) { + FileSourceDb.Test.Builder test = FileSourceDb.Test.newBuilder() + .setUuid(Uuids.create()) + .setName(RandomStringUtils.randomAlphanumeric(20)) + .setStatus(Constants.TestStatus.FAILURE) + .setStacktrace(RandomStringUtils.randomAlphanumeric(50)) + .setMsg(RandomStringUtils.randomAlphanumeric(30)) + .setExecutionTimeMs(RandomUtils.nextLong()); + int numberOfCoveredFiles = RandomUtils.nextInt(10); + for (int j = 0; j < numberOfCoveredFiles; j++) { + test.addCoveredFile( + FileSourceDb.Test.CoveredFile.newBuilder() + .setFileUuid(Uuids.create()) + .addCoveredLine(RandomUtils.nextInt(500)) + ); + } + tests.add(test.build()); + } + return tests; + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java index 54102e827e7..c02292fdf37 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java @@ -97,8 +97,7 @@ public class TestIndexTest { .setMessage("message-" + id) .setStackTrace("stacktrace-" + id) .setStatus("status-" + id) - .setType("type-" + id) - .setUuid("uuid-" + id) + .setFileUuid("uuid-" + id) .setCoverageBlocks(Arrays.asList(coverageBlocks)); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java new file mode 100644 index 00000000000..5356f4d1885 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java @@ -0,0 +1,203 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.test.index; + +import com.google.common.collect.Iterators; +import org.apache.commons.io.FileUtils; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.sonar.api.config.Settings; +import org.sonar.batch.protocol.Constants; +import org.sonar.core.persistence.DbTester; +import org.sonar.server.db.DbClient; +import org.sonar.server.es.EsTester; +import org.sonar.server.source.db.FileSourceDb; +import org.sonar.server.source.index.FileSourcesUpdaterUtil; +import org.sonar.server.test.db.TestTesting; +import org.sonar.test.DbTests; +import org.sonar.test.TestUtils; + +import java.io.IOException; +import java.sql.Connection; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.sonar.server.test.index.TestIndexDefinition.*; + +@Category(DbTests.class) +public class TestIndexerTest { + + @ClassRule + public static EsTester es = new EsTester().addDefinitions(new TestIndexDefinition(new Settings())); + + @ClassRule + public static DbTester db = new DbTester(); + + private TestIndexer sut; + + @Before + public void setUp() { + es.truncateIndices(); + db.truncateTables(); + sut = new TestIndexer(new DbClient(db.database(), db.myBatis()), es.client()); + sut.setEnabled(true); + } + + @Test + public void index_tests() throws Exception { + db.prepareDbUnit(getClass(), "db.xml"); + Connection connection = db.openConnection(); + TestTesting.updateDataColumn(connection, "FILE_UUID", TestTesting.newRandomTests(3)); + connection.close(); + + sut.index(); + + assertThat(countDocuments()).isEqualTo(3); + } + + @Test + public void index_tests_from_project() throws Exception { + db.prepareDbUnit(getClass(), "db.xml"); + + Connection connection = db.openConnection(); + TestTesting.updateDataColumn(connection, "FILE_UUID", TestTesting.newRandomTests(3)); + connection.close(); + + sut.index("PROJECT_UUID"); + assertThat(countDocuments()).isEqualTo(3); + } + + @Test + public void index_nothing_from_unknown_project() throws Exception { + db.prepareDbUnit(getClass(), "db.xml"); + + Connection connection = db.openConnection(); + TestTesting.updateDataColumn(connection, "FILE_UUID", TestTesting.newRandomTests(3)); + connection.close(); + + sut.index("UNKNOWN"); + assertThat(countDocuments()).isZero(); + } + + /** + * File F1 in project P1 has one test -> to be updated + * File F2 in project P1 has one test -> untouched + */ + + @Test + public void update_already_indexed_test() throws Exception { + indexTest("P1", "F1", "T1", "U111"); + indexTest("P1", "F2", "T1", "U121"); + + FileSourcesUpdaterUtil.Row dbRow = TestResultSetIterator.toRow("P1", "F1", new Date(), Arrays.asList( + FileSourceDb.Test.newBuilder() + .setUuid("U111") + .setName("NAME_1") + .setStatus(Constants.TestStatus.FAILURE) + .setMsg("NEW_MESSAGE_1") + .setStacktrace("NEW_STACKTRACE_1") + .setExecutionTimeMs(123_456L) + .addCoveredFile(FileSourceDb.Test.CoveredFile.newBuilder().setFileUuid("MAIN_UUID_1").addCoveredLine(42)) + .build() + )); + sut.index(Iterators.singletonIterator(dbRow)); + + assertThat(countDocuments()).isEqualTo(2L); + + SearchResponse fileSearch = prepareSearch() + .setQuery(QueryBuilders.termQuery(FIELD_FILE_UUID, "F1")) + .get(); + assertThat(fileSearch.getHits().getTotalHits()).isEqualTo(1L); + Map fields = fileSearch.getHits().getHits()[0].sourceAsMap(); + assertThat(fields).contains( + entry(FIELD_PROJECT_UUID, "P1"), + entry(FIELD_FILE_UUID, "F1"), + entry(FIELD_TEST_UUID, "U111"), + entry(FIELD_NAME, "NAME_1"), + entry(FIELD_STATUS, "FAILURE"), + entry(FIELD_MESSAGE, "NEW_MESSAGE_1"), + entry(FIELD_STACKTRACE, "NEW_STACKTRACE_1"), + entry(FIELD_DURATION_IN_MS, 123_456) + ); + } + + @Test + public void delete_file_uuid() throws Exception { + indexTest("P1", "F1", "T1", "U111"); + indexTest("P1", "F1", "T2", "U112"); + indexTest("P1", "F2", "T1", "U121"); + + sut.deleteByFile("F1"); + + List hits = getDocuments(); + Map document = hits.get(0).getSource(); + assertThat(hits).hasSize(1); + assertThat(document.get(FIELD_NAME)).isEqualTo("NAME_1"); + assertThat(document.get(FIELD_FILE_UUID)).isEqualTo("F2"); + } + + @Test + public void delete_by_project_uuid() throws Exception { + indexTest("P1", "F1", "T1", "U111"); + indexTest("P1", "F1", "T2", "U112"); + indexTest("P1", "F2", "T1", "U121"); + indexTest("P2", "F3", "T1", "U231"); + + sut.deleteByProject("P1"); + + List hits = getDocuments(); + assertThat(hits).hasSize(1); + Map document = hits.get(0).getSource(); + assertThat(hits).hasSize(1); + assertThat(document.get(FIELD_PROJECT_UUID)).isEqualTo("P2"); + } + + private void indexTest(String projectUuid, String fileUuid, String testName, String uuid) throws IOException { + es.client().prepareIndex(INDEX, TYPE) + .setId(uuid) + .setSource(FileUtils.readFileToString(TestUtils.getResource(this.getClass(), projectUuid + "_" + fileUuid + "_" + testName + ".json"))) + .setRefresh(true) + .get(); + } + + private SearchRequestBuilder prepareSearch() { + return es.client().prepareSearch(INDEX) + .setTypes(TYPE); + } + + private List getDocuments() { + return es.getDocuments(INDEX, TYPE); + } + + private long countDocuments() { + return es.countDocuments(INDEX, TYPE); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestResultSetIteratorTest.java new file mode 100644 index 00000000000..8aa38143d7e --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestResultSetIteratorTest.java @@ -0,0 +1,191 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.test.index; + +import org.assertj.core.data.MapEntry; +import org.elasticsearch.action.update.UpdateRequest; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.sonar.core.persistence.DbTester; +import org.sonar.server.db.DbClient; +import org.sonar.server.source.db.FileSourceDb; +import org.sonar.server.source.index.FileSourcesUpdaterUtil; +import org.sonar.server.source.index.SourceLineResultSetIteratorTest; +import org.sonar.server.test.db.TestTesting; +import org.sonar.test.DbTests; + +import java.sql.Connection; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +@Category(DbTests.class) +public class TestResultSetIteratorTest { + + @ClassRule + public static DbTester db = new DbTester().schema(SourceLineResultSetIteratorTest.class, "schema.sql"); + + DbClient dbClient; + + Connection connection; + + TestResultSetIterator sut; + + @Before + public void setUp() throws Exception { + dbClient = new DbClient(db.database(), db.myBatis()); + connection = db.openConnection(); + } + + @After + public void after() throws Exception { + if (sut != null) { + sut.close(); + } + connection.close(); + } + + @Test + public void traverse_db() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + TestTesting.updateDataColumn(connection, "F1", TestTesting.newFakeTests(3)); + sut = TestResultSetIterator.create(dbClient, connection, 0L, null); + + FileSourcesUpdaterUtil.Row row = sut.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); + assertThat(row.getUpdateRequests()).hasSize(3); + + UpdateRequest firstRequest = row.getUpdateRequests().get(0); + Map doc = firstRequest.doc().sourceAsMap(); + assertThat(doc).contains( + MapEntry.entry(TestIndexDefinition.FIELD_PROJECT_UUID, "P1"), + MapEntry.entry(TestIndexDefinition.FIELD_FILE_UUID, "F1"), + MapEntry.entry(TestIndexDefinition.FIELD_TEST_UUID, "TEST_FILE_UUID_1"), + MapEntry.entry(TestIndexDefinition.FIELD_STATUS, "FAILURE"), + MapEntry.entry(TestIndexDefinition.FIELD_MESSAGE, "MESSAGE_1"), + MapEntry.entry(TestIndexDefinition.FIELD_DURATION_IN_MS, 1), + MapEntry.entry(TestIndexDefinition.FIELD_STACKTRACE, "STACKTRACE_1"), + MapEntry.entry(TestIndexDefinition.FIELD_NAME, "NAME_1") + ); + } + + /** + * File with one line. No metadata available on the line. + */ + @Test + public void minimal_data() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + List tests = Arrays.asList( + FileSourceDb.Test.newBuilder() + .setUuid("U1") + .setName("N1") + .build() + ); + TestTesting.updateDataColumn(connection, "F1", tests); + sut = TestResultSetIterator.create(dbClient, connection, 0L, null); + + FileSourcesUpdaterUtil.Row row = sut.next(); + + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); + assertThat(row.getUpdateRequests()).hasSize(1); + UpdateRequest firstRequest = row.getUpdateRequests().get(0); + Map doc = firstRequest.doc().sourceAsMap(); + assertThat(doc).contains( + MapEntry.entry(TestIndexDefinition.FIELD_PROJECT_UUID, "P1"), + MapEntry.entry(TestIndexDefinition.FIELD_FILE_UUID, "F1"), + MapEntry.entry(TestIndexDefinition.FIELD_TEST_UUID, "U1"), + MapEntry.entry(TestIndexDefinition.FIELD_NAME, "N1") + ); + // null values + assertThat(doc).containsKeys( + TestIndexDefinition.FIELD_DURATION_IN_MS, + TestIndexDefinition.FIELD_STACKTRACE, + TestIndexDefinition.FIELD_MESSAGE, + TestIndexDefinition.FIELD_STATUS, + TestIndexDefinition.FIELD_COVERAGE_BLOCKS + ); + } + + @Test + public void filter_by_date() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + sut = TestResultSetIterator.create(dbClient, connection, 2000000000000L, null); + + assertThat(sut.hasNext()).isFalse(); + } + + @Test + public void filter_by_project() throws Exception { + db.prepareDbUnit(getClass(), "filter_by_project.xml"); + TestTesting.updateDataColumn(connection, "F1", TestTesting.newFakeTests(1)); + + sut = TestResultSetIterator.create(dbClient, connection, 0L, "P1"); + + FileSourcesUpdaterUtil.Row row = sut.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + + // File from other project P2 is not returned + assertThat(sut.hasNext()).isFalse(); + } + + @Test + public void filter_by_project_and_date() throws Exception { + db.prepareDbUnit(getClass(), "filter_by_project_and_date.xml"); + TestTesting.updateDataColumn(connection, "F1", TestTesting.newFakeTests(1)); + + sut = TestResultSetIterator.create(dbClient, connection, 1400000000000L, "P1"); + + FileSourcesUpdaterUtil.Row row = sut.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + + // File F2 is not returned + assertThat(sut.hasNext()).isFalse(); + } + + @Test + public void fail_on_bad_data_format() throws Exception { + db.prepareDbUnit(getClass(), "shared.xml"); + + TestTesting.updateDataColumn(connection, "F1", "THIS_IS_NOT_PROTOBUF".getBytes()); + + sut = TestResultSetIterator.create(dbClient, connection, 0L, null); + try { + assertThat(sut.hasNext()).isTrue(); + sut.next(); + fail("it should not be possible to go through not compliant data"); + } catch (IllegalStateException e) { + // ok + } + } + +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexTestsStepTest/index_source.xml b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexTestsStepTest/index_source.xml new file mode 100644 index 00000000000..ad0cb1a2f57 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexTestsStepTest/index_source.xml @@ -0,0 +1,11 @@ + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project.xml deleted file mode 100644 index 1f0032170a0..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project_and_date.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project_and_date.xml deleted file mode 100644 index e6289cd994c..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project_and_date.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/schema.sql deleted file mode 100644 index 99f823ff74d..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/schema.sql +++ /dev/null @@ -1,11 +0,0 @@ - -CREATE TABLE "FILE_SOURCES" ( - "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), - "PROJECT_UUID" VARCHAR(50) NOT NULL, - "FILE_UUID" VARCHAR(50) NOT NULL, - "BINARY_DATA" BINARY(167772150), - "DATA_HASH" VARCHAR(50) NOT NULL, - "DATA_TYPE" VARCHAR(50), - "CREATED_AT" BIGINT NOT NULL, - "UPDATED_AT" BIGINT NOT NULL -); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/shared.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/shared.xml deleted file mode 100644 index 053270327d6..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/shared.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/filter_by_project.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/filter_by_project.xml new file mode 100644 index 00000000000..1f0032170a0 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/filter_by_project.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/filter_by_project_and_date.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/filter_by_project_and_date.xml new file mode 100644 index 00000000000..e6289cd994c --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/filter_by_project_and_date.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql new file mode 100644 index 00000000000..99f823ff74d --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql @@ -0,0 +1,11 @@ + +CREATE TABLE "FILE_SOURCES" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "PROJECT_UUID" VARCHAR(50) NOT NULL, + "FILE_UUID" VARCHAR(50) NOT NULL, + "BINARY_DATA" BINARY(167772150), + "DATA_HASH" VARCHAR(50) NOT NULL, + "DATA_TYPE" VARCHAR(50), + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL +); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/shared.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/shared.xml new file mode 100644 index 00000000000..053270327d6 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/shared.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F1_T1.json b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F1_T1.json new file mode 100644 index 00000000000..52013fe2743 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F1_T1.json @@ -0,0 +1,11 @@ +{ + "projectUuid": "P1", + "fileUuid": "F1", + "testUuid": "U111", + "name": "NAME_1", + "status": "FAILURE", + "durationInMs": 100000, + "message": "MESSAGE_1", + "stacktrace": "STACKTRACE_1", + "updatedAt": "2014-01-01T23:45:01.8+01:00" +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F1_T2.json b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F1_T2.json new file mode 100644 index 00000000000..6a76a8054a5 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F1_T2.json @@ -0,0 +1,11 @@ +{ + "projectUuid": "P1", + "fileUuid": "F1", + "testUuid": "U112", + "name": "NAME_2", + "status": "FAILURE", + "durationInMs": 100000, + "message": "MESSAGE_1", + "stacktrace": "STACKTRACE_1", + "updatedAt": "2014-01-01T23:45:01.8+01:00" +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F2_T1.json b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F2_T1.json new file mode 100644 index 00000000000..1c2e2b9826c --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P1_F2_T1.json @@ -0,0 +1,11 @@ +{ + "projectUuid": "P1", + "fileUuid": "F2", + "testUuid": "U121", + "name": "NAME_1", + "status": "FAILURE", + "durationInMs": 100000, + "message": "MESSAGE_1", + "stacktrace": "STACKTRACE_1", + "updatedAt": "2014-01-01T23:45:01.8+01:00" +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P2_F3_T1.json b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P2_F3_T1.json new file mode 100644 index 00000000000..643e58b22d8 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/P2_F3_T1.json @@ -0,0 +1,11 @@ +{ + "projectUuid": "P2", + "fileUuid": "F3", + "testUuid": "U231", + "name": "NAME_1", + "status": "FAILURE", + "durationInMs": 100000, + "message": "MESSAGE_1", + "stacktrace": "STACKTRACE_1", + "updatedAt": "2014-01-01T23:45:01.8+01:00" +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/db.xml b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/db.xml new file mode 100644 index 00000000000..c2326dde5dc --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestIndexerTest/db.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/filter_by_project.xml b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/filter_by_project.xml new file mode 100644 index 00000000000..c629e0e50a8 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/filter_by_project.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/filter_by_project_and_date.xml b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/filter_by_project_and_date.xml new file mode 100644 index 00000000000..2edb2eafec1 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/filter_by_project_and_date.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/schema.sql new file mode 100644 index 00000000000..99f823ff74d --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/schema.sql @@ -0,0 +1,11 @@ + +CREATE TABLE "FILE_SOURCES" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "PROJECT_UUID" VARCHAR(50) NOT NULL, + "FILE_UUID" VARCHAR(50) NOT NULL, + "BINARY_DATA" BINARY(167772150), + "DATA_HASH" VARCHAR(50) NOT NULL, + "DATA_TYPE" VARCHAR(50), + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL +); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/shared.xml b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/shared.xml new file mode 100644 index 00000000000..3942d39e9f6 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/test/index/TestResultSetIteratorTest/shared.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml index f70496ea9ca..7c5430038ec 100644 --- a/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/component/db/ComponentMapper.xml @@ -178,7 +178,7 @@