From a17301de1b5050815a9219003f138330ffc707bb Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Tue, 17 Mar 2015 16:43:27 +0100 Subject: [PATCH] Batch should set updated date to 0 on updated sources and Compute should update this sources for current project --- .../step/IndexSourceLinesStep.java | 23 ++++++- .../step/IndexSourceLinesStepTest.java | 65 ++++++++++++++++++- ..._sources_with_update_at_to_zero-result.xml | 7 ++ ...date_on_sources_with_update_at_to_zero.xml | 7 ++ .../sonar/batch/index/SourcePersister.java | 4 +- .../sonar/core/source/db/FileSourceDao.java | 4 ++ .../core/source/db/FileSourceMapper.java | 4 ++ .../sonar/core/source/db/FileSourceMapper.xml | 15 +++-- .../core/source/db/FileSourceDaoTest.java | 10 +++ ..._date_when_updated_date_is_zero-result.xml | 18 +++++ .../update_date_when_updated_date_is_zero.xml | 16 +++++ 11 files changed, 163 insertions(+), 10 deletions(-) create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero.xml create mode 100644 sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/update_date_when_updated_date_is_zero-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/update_date_when_updated_date_is_zero.xml diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java index 7746f78c72a..46e8836c1af 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java @@ -20,14 +20,22 @@ package org.sonar.server.computation.step; import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; +import org.sonar.core.persistence.DbSession; +import org.sonar.core.persistence.MyBatis; import org.sonar.server.computation.ComputationContext; +import org.sonar.server.db.DbClient; import org.sonar.server.source.index.SourceLineIndexer; public class IndexSourceLinesStep implements ComputationStep { + private final DbClient dbClient; + private final System2 system2; private final SourceLineIndexer indexer; - public IndexSourceLinesStep(SourceLineIndexer indexer) { + public IndexSourceLinesStep(DbClient dbClient, System2 system2, SourceLineIndexer indexer) { + this.dbClient = dbClient; + this.system2 = system2; this.indexer = indexer; } @@ -38,9 +46,22 @@ public class IndexSourceLinesStep implements ComputationStep { @Override public void execute(ComputationContext context) { + updateSourceUpdateDate(context.getProject().uuid()); indexer.index(); } + // Temporary solution to only index in E/S updated sources from current project + // Should be removed when source wil be persisted in compute engine + private void updateSourceUpdateDate(String projectUuid){ + DbSession session = dbClient.openSession(true); + try { + dbClient.fileSourceDao().updateDateWhenUpdatedDateIsZero(session, projectUuid, system2.now()); + session.commit(); + } finally { + MyBatis.closeQuietly(session); + } + } + @Override public String getDescription() { return "Index source lines"; diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexSourceLinesStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexSourceLinesStepTest.java index fb0042ba9ea..8b5d1f9aad5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexSourceLinesStepTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/step/IndexSourceLinesStepTest.java @@ -19,22 +19,81 @@ */ package org.sonar.server.computation.step; +import org.elasticsearch.search.SearchHit; +import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; +import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.core.persistence.DbTester; +import org.sonar.core.source.db.FileSourceDao; +import org.sonar.server.component.ComponentTesting; +import org.sonar.server.computation.ComputationContext; +import org.sonar.server.db.DbClient; +import org.sonar.server.es.EsTester; +import org.sonar.server.source.db.FileSourceTesting; +import org.sonar.server.source.index.SourceLineDoc; +import org.sonar.server.source.index.SourceLineIndexDefinition; import org.sonar.server.source.index.SourceLineIndexer; import java.io.IOException; +import java.sql.Connection; +import java.util.Date; +import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class IndexSourceLinesStepTest extends BaseStepTest { - @Test - public void supportedProjectQualifiers() throws Exception { + @ClassRule + public static DbTester dbTester = new DbTester(); + + @ClassRule + public static EsTester esTester = new EsTester().addDefinitions(new SourceLineIndexDefinition(new Settings())); + + System2 system2; + + DbClient dbClient; + @Before + public void setUp() throws Exception { + dbClient = new DbClient(dbTester.database(), dbTester.myBatis(), new FileSourceDao(null)); + system2 = mock(System2.class); } @Override protected ComputationStep step() throws IOException { - return new IndexSourceLinesStep(mock(SourceLineIndexer.class)); + SourceLineIndexer sourceLineIndexer = new SourceLineIndexer(dbClient, esTester.client()); + sourceLineIndexer.setEnabled(true); + return new IndexSourceLinesStep(dbClient, system2, sourceLineIndexer); + } + + @Test + public void supported_project_qualifiers() throws Exception { + assertThat(step().supportedProjectQualifiers()).containsOnly(Qualifiers.PROJECT); + } + + @Test + public void update_source_date_on_sources_with_update_at_to_zero() throws Exception { + when(system2.now()).thenReturn(150000000002L); + dbTester.prepareDbUnit(getClass(), "update_source_date_on_sources_with_update_at_to_zero.xml"); + Connection connection = dbTester.openConnection(); + FileSourceTesting.updateDataColumn(connection, "FILE1_UUID", FileSourceTesting.newRandomData(1).build()); + connection.close(); + + step().execute(new ComputationContext(mock(BatchReportReader.class), ComponentTesting.newProjectDto("ABCD"))); + + dbTester.assertDbUnit(getClass(), "update_source_date_on_sources_with_update_at_to_zero-result.xml"); + + List docs = esTester.getDocuments(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE); + assertThat(docs).hasSize(1); + SourceLineDoc doc = new SourceLineDoc(docs.get(0).sourceAsMap()); + assertThat(doc.projectUuid()).isEqualTo("ABCD"); + assertThat(doc.fileUuid()).isEqualTo("FILE1_UUID"); + assertThat(doc.updateDate()).isEqualTo(new Date(system2.now())); } } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml new file mode 100644 index 00000000000..2a8f8a09034 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero-result.xml @@ -0,0 +1,7 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero.xml b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero.xml new file mode 100644 index 00000000000..31135717348 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/computation/step/IndexSourceLinesStepTest/update_source_date_on_sources_with_update_at_to_zero.xml @@ -0,0 +1,7 @@ + + + + + diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java index dd084a834c2..e27d9eca168 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java @@ -102,7 +102,7 @@ public class SourcePersister implements ScanPersister { .setSrcHash(metadata.hash()) .setLineHashes(lineHashesAsMd5Hex(inputFile)) .setCreatedAt(system2.now()) - .setUpdatedAt(system2.now()); + .setUpdatedAt(0L); mapper.insert(dto); session.commit(); } else { @@ -115,7 +115,7 @@ public class SourcePersister implements ScanPersister { .setLineHashes(lineHashesAsMd5Hex(inputFile)); // Optimization do not change updated at when updating src_hash to avoid indexation by E/S if (!dataHash.equals(previousDto.getDataHash())) { - previousDto.setUpdatedAt(system2.now()); + previousDto.setUpdatedAt(0L); } mapper.update(previousDto); session.commit(); diff --git a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java index eced8499a4d..16864674f20 100644 --- a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java +++ b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDao.java @@ -121,4 +121,8 @@ public class FileSourceDao implements BatchComponent, ServerComponent, DaoCompon } } + public void updateDateWhenUpdatedDateIsZero(DbSession session, String projectUuid, long updateDate) { + session.getMapper(FileSourceMapper.class).updateDateWhenUpdatedDateIsZero(projectUuid, updateDate); + } + } diff --git a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceMapper.java b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceMapper.java index e8876892357..633e4d140ed 100644 --- a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceMapper.java @@ -20,6 +20,8 @@ package org.sonar.core.source.db; +import org.apache.ibatis.annotations.Param; + import javax.annotation.CheckForNull; import java.util.List; @@ -35,6 +37,8 @@ public interface FileSourceMapper { void update(FileSourceDto dto); + void updateDateWhenUpdatedDateIsZero(@Param("projectUuid") String projectUuid, @Param("date") Long updateDate); + @CheckForNull String selectLineHashes(String fileUuid); } diff --git a/sonar-core/src/main/resources/org/sonar/core/source/db/FileSourceMapper.xml b/sonar-core/src/main/resources/org/sonar/core/source/db/FileSourceMapper.xml index 2edae7f055c..49ede1dfb30 100644 --- a/sonar-core/src/main/resources/org/sonar/core/source/db/FileSourceMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/source/db/FileSourceMapper.xml @@ -18,20 +18,27 @@ - insert into file_sources (project_uuid, file_uuid, created_at, updated_at, binary_data, line_hashes, data_hash, src_hash) - values (#{projectUuid,jdbcType=VARCHAR}, #{fileUuid,jdbcType=VARCHAR}, #{createdAt,jdbcType=BIGINT}, + INSERT INTO file_sources (project_uuid, file_uuid, created_at, updated_at, binary_data, line_hashes, data_hash, src_hash) + VALUES (#{projectUuid,jdbcType=VARCHAR}, #{fileUuid,jdbcType=VARCHAR}, #{createdAt,jdbcType=BIGINT}, #{updatedAt,jdbcType=BIGINT}, #{binaryData,jdbcType=BLOB}, #{lineHashes,jdbcType=CLOB}, #{dataHash,jdbcType=VARCHAR}, #{srcHash,jdbcType=VARCHAR}) - update file_sources set + UPDATE file_sources SET updated_at = #{updatedAt,jdbcType=BIGINT}, binary_data = #{binaryData,jdbcType=BLOB}, line_hashes = #{lineHashes,jdbcType=CLOB}, data_hash = #{dataHash,jdbcType=VARCHAR}, src_hash = #{srcHash,jdbcType=VARCHAR} - where id = #{id} + WHERE id = #{id} + + + + UPDATE file_sources SET + updated_at = #{date,jdbcType=BIGINT} + WHERE project_uuid = #{projectUuid} + AND updated_at = 0