From 803a3432fd937479db19aff857a9c9357ccca5fe Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 16 Apr 2015 18:17:57 +0200 Subject: [PATCH] SONAR-6258 Remove hack setting updated_at to 0 by only indexing file sources of current project --- .../benchmark/SourceDbBenchmarkTest.java | 2 +- .../computation/source/SymbolsLineReader.java | 21 ++++--- .../step/IndexSourceLinesStep.java | 25 +------- .../step/PersistFileSourcesStep.java | 22 ++++--- .../java/org/sonar/server/es/BaseIndexer.java | 19 +++++- .../index/SourceFileResultSetIterator.java | 60 +++++++++++++----- .../source/index/SourceLineIndexer.java | 17 +++++- .../step/IndexSourceLinesStepTest.java | 16 +---- .../step/PersistFileSourcesStepTest.java | 10 ++- .../index/IssueAuthorizationIndexerTest.java | 2 +- .../SourceFileResultSetIteratorTest.java | 61 +++++++++++++++---- .../source/index/SourceLineIndexerTest.java | 45 +++++++++++++- ...update_at_to_zero.xml => index_source.xml} | 4 ++ ..._sources_with_update_at_to_zero-result.xml | 8 --- .../filter_by_project.xml | 9 +++ .../filter_by_project_and_date.xml | 9 +++ .../server/source/db/FileSourceTestDb.java | 48 +++++++++++++++ 17 files changed, 277 insertions(+), 101 deletions(-) rename server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/{update_source_date_on_sources_with_update_at_to_zero.xml => index_source.xml} (55%) delete mode 100644 server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project_and_date.xml diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java index d29374d2f57..9bc46b01d20 100644 --- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java +++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java @@ -73,7 +73,7 @@ public class SourceDbBenchmarkTest { try { long start = System.currentTimeMillis(); - SourceFileResultSetIterator it = SourceFileResultSetIterator.create(dbClient, connection, 0L); + SourceFileResultSetIterator it = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); while (it.hasNext()) { SourceFileResultSetIterator.Row row = it.next(); assertThat(row.getLineUpdateRequests().size()).isEqualTo(NUMBER_OF_LINES); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/source/SymbolsLineReader.java b/server/sonar-server/src/main/java/org/sonar/server/computation/source/SymbolsLineReader.java index 9045e0eb9a3..db57bcb52c8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/source/SymbolsLineReader.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/source/SymbolsLineReader.java @@ -24,11 +24,16 @@ import org.sonar.batch.protocol.output.BatchReport; import org.sonar.server.source.db.FileSourceDb; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import static com.google.common.collect.Lists.newArrayList; -import static com.google.common.collect.Maps.newHashMap; -import static com.google.common.collect.Sets.newHashSet; public class SymbolsLineReader implements LineReader { @@ -39,7 +44,7 @@ public class SymbolsLineReader implements LineReader { this.symbols = newArrayList(symbols); // Sort symbols to have deterministic results and avoid false variation that would lead to an unnecessary update of the source files // data - Collections.sort(this.symbols, new SymbolsDuplication()); + Collections.sort(this.symbols, new SymbolsComparator()); this.idsBySymbol = createIdsBySymbolMap(this.symbols); } @@ -78,8 +83,8 @@ public class SymbolsLineReader implements LineReader { } private List findSymbolsMatchingLine(int line) { - List lineSymbols = newArrayList(); - Set symbolsIndex = newHashSet(); + List lineSymbols = new ArrayList<>(); + Set symbolsIndex = new HashSet<>(); for (BatchReport.Symbols.Symbol symbol : symbols) { if (matchLine(symbol.getDeclaration(), line) && !symbolsIndex.contains(symbol)) { lineSymbols.add(symbol); @@ -101,7 +106,7 @@ public class SymbolsLineReader implements LineReader { } private Map createIdsBySymbolMap(List symbols) { - Map map = newHashMap(); + Map map = new HashMap<>(); int symbolId = 1; for (BatchReport.Symbols.Symbol symbol : symbols) { map.put(symbol, symbolId); @@ -110,7 +115,7 @@ public class SymbolsLineReader implements LineReader { return map; } - private static class SymbolsDuplication implements Comparator, Serializable { + private static class SymbolsComparator implements Comparator, Serializable { @Override public int compare(BatchReport.Symbols.Symbol o1, BatchReport.Symbols.Symbol o2) { if (o1.getDeclaration().getStartLine() == o2.getDeclaration().getStartLine()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java index 46e8836c1af..8bcc854162b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java @@ -20,22 +20,14 @@ package org.sonar.server.computation.step; import org.sonar.api.resources.Qualifiers; -import org.sonar.api.utils.System2; -import org.sonar.core.persistence.DbSession; -import org.sonar.core.persistence.MyBatis; import org.sonar.server.computation.ComputationContext; -import org.sonar.server.db.DbClient; import org.sonar.server.source.index.SourceLineIndexer; public class IndexSourceLinesStep implements ComputationStep { - private final DbClient dbClient; - private final System2 system2; private final SourceLineIndexer indexer; - public IndexSourceLinesStep(DbClient dbClient, System2 system2, SourceLineIndexer indexer) { - this.dbClient = dbClient; - this.system2 = system2; + public IndexSourceLinesStep(SourceLineIndexer indexer) { this.indexer = indexer; } @@ -46,20 +38,7 @@ public class IndexSourceLinesStep implements ComputationStep { @Override public void execute(ComputationContext context) { - updateSourceUpdateDate(context.getProject().uuid()); - indexer.index(); - } - - // Temporary solution to only index in E/S updated sources from current project - // Should be removed when source wil be persisted in compute engine - private void updateSourceUpdateDate(String projectUuid){ - DbSession session = dbClient.openSession(true); - try { - dbClient.fileSourceDao().updateDateWhenUpdatedDateIsZero(session, projectUuid, system2.now()); - session.commit(); - } finally { - MyBatis.closeQuietly(session); - } + indexer.index(context.getProject().uuid()); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java index de067227d8b..974a5678a49 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java @@ -36,18 +36,24 @@ import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.MyBatis; import org.sonar.core.source.db.FileSourceDto; import org.sonar.server.computation.ComputationContext; -import org.sonar.server.computation.source.*; +import org.sonar.server.computation.source.ComputeFileSourceData; +import org.sonar.server.computation.source.CoverageLineReader; +import org.sonar.server.computation.source.DuplicationLineReader; +import org.sonar.server.computation.source.HighlightingLineReader; +import org.sonar.server.computation.source.LineReader; +import org.sonar.server.computation.source.ReportIterator; +import org.sonar.server.computation.source.ScmLineReader; +import org.sonar.server.computation.source.SymbolsLineReader; import org.sonar.server.db.DbClient; import org.sonar.server.source.db.FileSourceDb; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.google.common.collect.Lists.newArrayList; - public class PersistFileSourcesStep implements ComputationStep { private final DbClient dbClient; @@ -131,8 +137,7 @@ public class PersistFileSourcesStep implements ComputationStep { .setDataHash(dataHash) .setLineHashes(lineHashes) .setCreatedAt(system2.now()) - // TODO set current date here when indexing sources in E/S will be done in this class - .setUpdatedAt(0L); + .setUpdatedAt(system2.now()); dbClient.fileSourceDao().insert(fileSourcesContext.session, dto); fileSourcesContext.session.commit(); } else { @@ -147,8 +152,7 @@ public class PersistFileSourcesStep implements ComputationStep { .setLineHashes(lineHashes); // Optimization only change updated at when updating binary data to avoid unecessary indexation by E/S if (binaryDataUpdated) { - // TODO set current date here when indexing sources in E/S will be done in this class - previousDto.setUpdatedAt(0L); + previousDto.setUpdatedAt(system2.now()); } dbClient.fileSourceDao().update(previousDto); fileSourcesContext.session.commit(); @@ -169,8 +173,8 @@ public class PersistFileSourcesStep implements ComputationStep { } private static class LineReaders { - private final List lineReaders = newArrayList(); - private final List reportIterators = newArrayList(); + private final List lineReaders = new ArrayList<>(); + private final List reportIterators = new ArrayList<>(); LineReaders(BatchReportReader reportReader, int componentRef) { File coverageFile = reportReader.readComponentCoverage(componentRef); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java index 39a0f443e89..548437c11ee 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java @@ -49,7 +49,7 @@ public abstract class BaseIndexer implements ServerComponent, Startable { private boolean enabled = false; protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName, - String dateFieldName) { + String dateFieldName) { this.indexName = indexName; this.typeName = typeName; this.dateFieldName = dateFieldName; @@ -58,14 +58,14 @@ public abstract class BaseIndexer implements ServerComponent, Startable { threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue()); } - public void index() { + public void index(final IndexerTask task) { if (enabled) { final long requestedAt = System.currentTimeMillis(); Future submit = executor.submit(new Runnable() { @Override public void run() { if (requestedAt > lastUpdatedAt) { - long l = doIndex(lastUpdatedAt); + long l = task.index(lastUpdatedAt); // l can be 0 if no documents were indexed lastUpdatedAt = Math.max(l, lastUpdatedAt); } @@ -79,6 +79,15 @@ public abstract class BaseIndexer implements ServerComponent, Startable { } } + public void index() { + index(new IndexerTask() { + @Override + public long index(long lastUpdatedAt) { + return doIndex(lastUpdatedAt); + } + }); + } + protected abstract long doIndex(long lastUpdatedAt); public BaseIndexer setEnabled(boolean b) { @@ -96,4 +105,8 @@ public abstract class BaseIndexer implements ServerComponent, Startable { executor.shutdown(); } + public interface IndexerTask { + long index(long lastUpdatedAt); + } + } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java index 3158d3cef7a..5403a6b9d54 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java @@ -29,6 +29,8 @@ import org.sonar.server.db.ResultSetIterator; import org.sonar.server.es.EsUtils; import org.sonar.server.source.db.FileSourceDb; +import javax.annotation.Nullable; + import java.io.ByteArrayOutputStream; import java.io.OutputStreamWriter; import java.sql.Connection; @@ -79,16 +81,23 @@ public class SourceFileResultSetIterator extends ResultSetIterator?"; - public static SourceFileResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) { + private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM file_sources "; + private static final String AFTER_DATE_FILTER = "updated_at>? "; + private static final String PROJECT_FILTER = "project_uuid=? "; + + public static SourceFileResultSetIterator create(DbClient dbClient, Connection connection, long afterDate, @Nullable String projectUuid) { try { - String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL; + String sql = createSQL(afterDate, projectUuid); // rows are big, so they are scrolled once at a time (one row in memory at a time) PreparedStatement stmt = dbClient.newScrollingSingleRowSelectStatement(connection, sql); + int index = 1; if (afterDate > 0L) { - stmt.setLong(1, afterDate); + stmt.setLong(index, afterDate); + index++; + } + if (projectUuid != null) { + stmt.setString(index, projectUuid); } return new SourceFileResultSetIterator(stmt); } catch (SQLException e) { @@ -96,6 +105,25 @@ public class SourceFileResultSetIterator extends ResultSetIterator 0L || projectUuid != null) { + sql += "WHERE "; + boolean isFirst = true; + if (afterDate > 0L) { + sql += AFTER_DATE_FILTER; + isFirst = false; + } + if (projectUuid != null) { + if (!isFirst) { + sql += "AND "; + } + sql += PROJECT_FILTER; + } + } + return sql; + } + private SourceFileResultSetIterator(PreparedStatement stmt) throws SQLException { super(stmt); } @@ -130,62 +158,62 @@ public class SourceFileResultSetIterator extends ResultSetIterator docs = esTester.getDocuments(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE); assertThat(docs).hasSize(1); SourceLineDoc doc = new SourceLineDoc(docs.get(0).sourceAsMap()); assertThat(doc.projectUuid()).isEqualTo("ABCD"); assertThat(doc.fileUuid()).isEqualTo("FILE1_UUID"); - assertThat(doc.updateDate()).isEqualTo(new Date(system2.now())); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java index 717b1030528..dc503775f8a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/step/PersistFileSourcesStepTest.java @@ -22,7 +22,11 @@ package org.sonar.server.computation.step; import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; -import org.junit.*; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import org.sonar.api.utils.System2; @@ -113,7 +117,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest { assertThat(fileSourceDto.getDataHash()).isNotEmpty(); assertThat(fileSourceDto.getLineHashes()).isNotEmpty(); assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now); - assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(0L); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now); FileSourceDb.Data data = FileSourceDto.decodeData(fileSourceDto.getBinaryData()); assertThat(data.getLinesCount()).isEqualTo(2); @@ -374,7 +378,7 @@ public class PersistFileSourcesStepTest extends BaseStepTest { assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); FileSourceDto fileSourceDto = dbClient.fileSourceDao().select(FILE_UUID); assertThat(fileSourceDto.getCreatedAt()).isEqualTo(past); - assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(0L); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueAuthorizationIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueAuthorizationIndexerTest.java index f77ba3148b1..6bfe8928e9b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueAuthorizationIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueAuthorizationIndexerTest.java @@ -54,7 +54,7 @@ public class IssueAuthorizationIndexerTest { @Test public void index_nothing() throws Exception { IssueAuthorizationIndexer indexer = createIndexer(); - indexer.doIndex(0L); + indexer.index(); assertThat(esTester.countDocuments("issues", "authorization")).isZero(); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceFileResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceFileResultSetIteratorTest.java index 6775d2845b3..1b6942ffdfc 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceFileResultSetIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceFileResultSetIteratorTest.java @@ -48,6 +48,8 @@ public class SourceFileResultSetIteratorTest { Connection connection; + SourceFileResultSetIterator iterator; + @Before public void setUp() throws Exception { dbClient = new DbClient(db.database(), db.myBatis()); @@ -56,6 +58,9 @@ public class SourceFileResultSetIteratorTest { @After public void after() throws Exception { + if (iterator != null) { + iterator.close(); + } connection.close(); } @@ -64,9 +69,9 @@ public class SourceFileResultSetIteratorTest { db.prepareDbUnit(getClass(), "shared.xml"); FileSourceTesting.updateDataColumn(connection, "F1", FileSourceTesting.newFakeData(3).build()); - SourceFileResultSetIterator it = SourceFileResultSetIterator.create(dbClient, connection, 0L); - assertThat(it.hasNext()).isTrue(); - SourceFileResultSetIterator.Row row = it.next(); + iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); + assertThat(iterator.hasNext()).isTrue(); + SourceFileResultSetIterator.Row row = iterator.next(); assertThat(row.getProjectUuid()).isEqualTo("P1"); assertThat(row.getFileUuid()).isEqualTo("F1"); assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); @@ -91,8 +96,7 @@ public class SourceFileResultSetIteratorTest { MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, 7), MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, 8), MapEntry.entry(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS, 9) - ); - it.close(); + ); } /** @@ -105,8 +109,8 @@ public class SourceFileResultSetIteratorTest { dataBuilder.addLinesBuilder().setLine(1).build(); FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); - SourceFileResultSetIterator it = SourceFileResultSetIterator.create(dbClient, connection, 0L); - SourceFileResultSetIterator.Row row = it.next(); + iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); + SourceFileResultSetIterator.Row row = iterator.next(); assertThat(row.getProjectUuid()).isEqualTo("P1"); assertThat(row.getFileUuid()).isEqualTo("F1"); assertThat(row.getUpdatedAt()).isEqualTo(1416239042000L); @@ -117,7 +121,7 @@ public class SourceFileResultSetIteratorTest { MapEntry.entry(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "P1"), MapEntry.entry(SourceLineIndexDefinition.FIELD_FILE_UUID, "F1"), MapEntry.entry(SourceLineIndexDefinition.FIELD_LINE, 1) - ); + ); // null values assertThat(doc).containsKeys( SourceLineIndexDefinition.FIELD_SCM_REVISION, @@ -134,16 +138,48 @@ public class SourceFileResultSetIteratorTest { SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS ); - it.close(); } @Test public void filter_by_date() throws Exception { db.prepareDbUnit(getClass(), "shared.xml"); - SourceFileResultSetIterator iterator = SourceFileResultSetIterator.create(dbClient, connection, 2000000000000L); + iterator = SourceFileResultSetIterator.create(dbClient, connection, 2000000000000L, null); + assertThat(iterator.hasNext()).isFalse(); + } + + @Test + public void filter_by_project() throws Exception { + db.prepareDbUnit(getClass(), "filter_by_project.xml"); + FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); + dataBuilder.addLinesBuilder().setLine(1).build(); + FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); + + iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, "P1"); + + SourceFileResultSetIterator.Row row = iterator.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + + // File from other project P2 is not returned + assertThat(iterator.hasNext()).isFalse(); + } + + @Test + public void filter_by_project_and_date() throws Exception { + db.prepareDbUnit(getClass(), "filter_by_project_and_date.xml"); + FileSourceDb.Data.Builder dataBuilder = FileSourceDb.Data.newBuilder(); + dataBuilder.addLinesBuilder().setLine(1).build(); + FileSourceTesting.updateDataColumn(connection, "F1", dataBuilder.build()); + + iterator = SourceFileResultSetIterator.create(dbClient, connection, 1400000000000L, "P1"); + + SourceFileResultSetIterator.Row row = iterator.next(); + assertThat(row.getProjectUuid()).isEqualTo("P1"); + assertThat(row.getFileUuid()).isEqualTo("F1"); + + // File F2 is not returned assertThat(iterator.hasNext()).isFalse(); - iterator.close(); } @Test @@ -152,7 +188,7 @@ public class SourceFileResultSetIteratorTest { FileSourceTesting.updateDataColumn(connection, "F1", "THIS_IS_NOT_PROTOBUF".getBytes()); - SourceFileResultSetIterator iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L); + iterator = SourceFileResultSetIterator.create(dbClient, connection, 0L, null); try { assertThat(iterator.hasNext()).isTrue(); iterator.next(); @@ -160,6 +196,5 @@ public class SourceFileResultSetIteratorTest { } catch (IllegalStateException e) { // ok } - iterator.close(); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java index 14b6c10b1a7..2a0952307c0 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java @@ -48,7 +48,24 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; -import static org.sonar.server.source.index.SourceLineIndexDefinition.*; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_DUPLICATIONS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_FILE_UUID; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_IT_CONDITIONS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_IT_LINE_HITS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_LINE; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_PROJECT_UUID; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_SCM_AUTHOR; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_SCM_REVISION; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_SOURCE; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_UT_CONDITIONS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_UT_LINE_HITS; +import static org.sonar.server.source.index.SourceLineIndexDefinition.INDEX; +import static org.sonar.server.source.index.SourceLineIndexDefinition.TYPE; @Category(DbTests.class) public class SourceLineIndexerTest { @@ -70,7 +87,7 @@ public class SourceLineIndexerTest { } @Test - public void index_source_lines_from_db() throws Exception { + public void index_source_lines() throws Exception { db.prepareDbUnit(getClass(), "db.xml"); Connection connection = db.openConnection(); @@ -81,6 +98,30 @@ public class SourceLineIndexerTest { assertThat(countDocuments()).isEqualTo(3); } + @Test + public void index_source_lines_from_project() throws Exception { + db.prepareDbUnit(getClass(), "db.xml"); + + Connection connection = db.openConnection(); + FileSourceTesting.updateDataColumn(connection, "FILE_UUID", FileSourceTesting.newRandomData(3).build()); + connection.close(); + + indexer.index("PROJECT_UUID"); + assertThat(countDocuments()).isEqualTo(3); + } + + @Test + public void index_nothing_from_unknown_project() throws Exception { + db.prepareDbUnit(getClass(), "db.xml"); + + Connection connection = db.openConnection(); + FileSourceTesting.updateDataColumn(connection, "FILE_UUID", FileSourceTesting.newRandomData(3).build()); + connection.close(); + + indexer.index("UNKNOWN"); + assertThat(countDocuments()).isZero(); + } + /** * File F1 in project P1 has one line -> to be updated * File F2 in project P1 has one line -> untouched diff --git a/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero.xml b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/index_source.xml similarity index 55% rename from server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero.xml rename to server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/index_source.xml index 63608752e31..bfa7d651386 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/index_source.xml @@ -5,4 +5,8 @@ test_data="[null]" created_at="1500000000000" updated_at="0"/> + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml deleted file mode 100644 index db05b2c2561..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project.xml new file mode 100644 index 00000000000..aeeffbdc05a --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project_and_date.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project_and_date.xml new file mode 100644 index 00000000000..db7a9977db3 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceFileResultSetIteratorTest/filter_by_project_and_date.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/sonar-batch-protocol/src/main/gen-java/org/sonar/server/source/db/FileSourceTestDb.java b/sonar-batch-protocol/src/main/gen-java/org/sonar/server/source/db/FileSourceTestDb.java index ac69f62aa29..47c041fb8c4 100644 --- a/sonar-batch-protocol/src/main/gen-java/org/sonar/server/source/db/FileSourceTestDb.java +++ b/sonar-batch-protocol/src/main/gen-java/org/sonar/server/source/db/FileSourceTestDb.java @@ -278,14 +278,26 @@ public final class FileSourceTestDb { /** * optional string key = 2; + * + * 
+       *TODO TBE - should the key and long_name specified directly ?
+       *
*/ boolean hasKey(); /** * optional string key = 2; + * + * 
+       *TODO TBE - should the key and long_name specified directly ?
+       *
*/ java.lang.String getKey(); /** * optional string key = 2; + * + * 
+       *TODO TBE - should the key and long_name specified directly ?
+       *
*/ com.google.protobuf.ByteString getKeyBytes(); @@ -511,12 +523,20 @@ public final class FileSourceTestDb { private java.lang.Object key_; /** * optional string key = 2; + * + * 
+       *TODO TBE - should the key and long_name specified directly ?
+       *
*/ public boolean hasKey() { return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional string key = 2; + * + * 
+       *TODO TBE - should the key and long_name specified directly ?
+       *
*/ public java.lang.String getKey() { java.lang.Object ref = key_; @@ -534,6 +554,10 @@ public final class FileSourceTestDb { } /** * optional string key = 2; + * + * 
+       *TODO TBE - should the key and long_name specified directly ?
+       *
*/ public com.google.protobuf.ByteString getKeyBytes() { @@ -1023,12 +1047,20 @@ public final class FileSourceTestDb { private java.lang.Object key_ = ""; /** * optional string key = 2; + * + * 
+         *TODO TBE - should the key and long_name specified directly ?
+         *
*/ public boolean hasKey() { return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional string key = 2; + * + * 
+         *TODO TBE - should the key and long_name specified directly ?
+         *
*/ public java.lang.String getKey() { java.lang.Object ref = key_; @@ -1046,6 +1078,10 @@ public final class FileSourceTestDb { } /** * optional string key = 2; + * + * 
+         *TODO TBE - should the key and long_name specified directly ?
+         *
*/ public com.google.protobuf.ByteString getKeyBytes() { @@ -1062,6 +1098,10 @@ public final class FileSourceTestDb { } /** * optional string key = 2; + * + * 
+         *TODO TBE - should the key and long_name specified directly ?
+         *
*/ public Builder setKey( java.lang.String value) { @@ -1075,6 +1115,10 @@ public final class FileSourceTestDb { } /** * optional string key = 2; + * + * 
+         *TODO TBE - should the key and long_name specified directly ?
+         *
*/ public Builder clearKey() { bitField0_ = (bitField0_ & ~0x00000002); @@ -1084,6 +1128,10 @@ public final class FileSourceTestDb { } /** * optional string key = 2; + * + * 
+         *TODO TBE - should the key and long_name specified directly ?
+         *
*/ public Builder setKeyBytes( com.google.protobuf.ByteString value) { -- 2.39.5