From f7598fbe9b8df747b525e9642fba3e7a3ae9da77 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lievremont Date: Mon, 24 Nov 2014 17:12:25 +0100 Subject: [PATCH] SONAR-5826 Use (big) integer column instead of DB timestamp for file_sources --- .../server/db/migrations/v50/FeedFileSources.java | 4 ++-- .../server/source/index/SourceLineIndexer.java | 3 +++ .../source/index/SourceLineResultSetIterator.java | 8 ++++---- .../index/SourceLineResultSetIteratorTest.java | 2 +- .../FeedFileSourcesMigrationTest/after-with-scm.xml | 2 +- .../v50/FeedFileSourcesMigrationTest/after.xml | 2 +- .../v50/FeedFileSourcesMigrationTest/schema.sql | 4 ++-- .../SourceLineResultSetIteratorTest/schema.sql | 4 ++-- .../source-with-scm.xml | 2 +- .../WEB-INF/db/migrate/713_create_file_sources.rb | 4 ++-- .../java/org/sonar/batch/index/SourcePersister.java | 13 ++++--------- .../org/sonar/batch/index/SourcePersisterTest.java | 12 ++++++------ .../index/SourcePersisterTest/file_sources.xml | 2 +- .../testPersistDontTouchUnchanged-result.xml | 2 +- .../testPersistEmptyFile-result.xml | 4 ++-- .../org/sonar/core/source/db/FileSourceDto.java | 13 ++++++------- .../org/sonar/core/persistence/schema-h2.ddl | 4 ++-- .../org/sonar/core/source/db/FileSourceDaoTest.java | 12 +++++++----- .../source/db/FileSourceDaoTest/insert-result.xml | 4 ++-- .../core/source/db/FileSourceDaoTest/shared.xml | 2 +- .../source/db/FileSourceDaoTest/update-result.xml | 2 +- 21 files changed, 52 insertions(+), 53 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java index 515a6fde99e..3c09e05947f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java @@ -60,8 +60,8 @@ public class FeedFileSources extends BaseDataChange { update.setString(1, projectUuid) .setString(2, fileUuid) - .setDate(3, now) - .setDate(4, updatedAt == null ? now : updatedAt) + .setLong(3, now.getTime()) + .setLong(4, (updatedAt == null ? now : updatedAt).getTime()) .setString(5, sourceData) .setString(6, ""); diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java index 2895663d2cc..52f758df34c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java @@ -92,6 +92,9 @@ public class SourceLineIndexer implements ServerComponent { // use cache. Will not work with Tomcat cluster. result = lastUpdatedAt; } + System.out.println("\n\n" + + " Checking for source lines > " + result + + "\n\n"); return result; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java index a89a0ae534a..86a22f2dc36 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java @@ -37,7 +37,6 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Timestamp; import java.util.Collection; import java.util.Date; import java.util.List; @@ -60,15 +59,16 @@ class SourceLineResultSetIterator extends ResultSetIterator=?"; + private static final String SQL_AFTER_DATE = SQL_ALL + " where updated_at>?"; static SourceLineResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) { try { String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL; PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql); if (afterDate > 0L) { - stmt.setTimestamp(1, new Timestamp(afterDate)); + stmt.setLong(1, afterDate); } + System.out.println(sql); return new SourceLineResultSetIterator(stmt); } catch (SQLException e) { throw new IllegalStateException("Fail to prepare SQL request to select all file sources", e); @@ -83,7 +83,7 @@ class SourceLineResultSetIterator extends ResultSetIterator read(ResultSet rs) throws SQLException { String projectUuid = rs.getString(1); String fileUuid = rs.getString(2); - Date updatedAt = SqlUtil.getDate(rs, 4); + Date updatedAt = new Date(SqlUtil.getLong(rs, 4)); int line = 1; List lines = Lists.newArrayList(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java index e58ad44f63d..7ca40e26238 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java @@ -87,7 +87,7 @@ public class SourceLineResultSetIteratorTest { db.prepareDbUnit(getClass(), "source-with-scm.xml"); SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, db.openConnection(), - DateUtils.parseDateTime("2014-11-01T16:44:02+0100").getTime()); + DateUtils.parseDateTime("2014-11-17T16:44:02+0100").getTime()); assertThat(iterator.hasNext()).isFalse(); } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after-with-scm.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after-with-scm.xml index 6d17e401179..c3948bde3b1 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after-with-scm.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after-with-scm.xml @@ -1,6 +1,6 @@ - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after.xml index 5369e25ab2d..b5de4a9cbfe 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after.xml @@ -1,6 +1,6 @@ - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/schema.sql index 0a99ce5b1d0..658181774d5 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/schema.sql +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/schema.sql @@ -113,6 +113,6 @@ CREATE TABLE "FILE_SOURCES" ( "FILE_UUID" VARCHAR(50) NOT NULL, "DATA" CLOB(2147483647), "DATA_HASH" VARCHAR(50) NOT NULL, - "CREATED_AT" TIMESTAMP NOT NULL, - "UPDATED_AT" TIMESTAMP NOT NULL + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL ); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql index d045f1f012b..3032e93f256 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql @@ -5,6 +5,6 @@ CREATE TABLE "FILE_SOURCES" ( "FILE_UUID" VARCHAR(50) NOT NULL, "DATA" CLOB(2147483647), "DATA_HASH" VARCHAR(50) NOT NULL, - "CREATED_AT" TIMESTAMP NOT NULL, - "UPDATED_AT" TIMESTAMP NOT NULL + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL ); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml index 3f2161acff0..f1498ba9c56 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml @@ -1,6 +1,6 @@ - diff --git a/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/713_create_file_sources.rb b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/713_create_file_sources.rb index d8cdc1b2776..422f355f95b 100644 --- a/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/713_create_file_sources.rb +++ b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/713_create_file_sources.rb @@ -29,8 +29,8 @@ class CreateFileSources < ActiveRecord::Migration t.column :file_uuid, :string, :limit => 50, :null => false t.column :data, :text, :null => true t.column :data_hash, :string, :limit => 50, :null => true - t.column :created_at, :datetime, :null => false - t.column :updated_at, :datetime, :null => false + t.column :created_at, :integer, :limit => 8, :null => false + t.column :updated_at, :integer, :limit => 8, :null => false end if dialect()=='mysql' diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java index b0f1199117e..6e42a07c18e 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java @@ -56,12 +56,7 @@ import javax.annotation.Nullable; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.google.common.base.Charsets.UTF_8; @@ -154,13 +149,13 @@ public class SourcePersister implements ScanPersister { if (previous == null) { FileSourceDto newFileSource = new FileSourceDto().setProjectUuid(projectTree.getRootProject().getUuid()).setFileUuid(fileUuid).setData(newData) .setDataHash(newDataHash) - .setCreatedAt(now) - .setUpdatedAt(now); + .setCreatedAt(now.getTime()) + .setUpdatedAt(now.getTime()); mapper.insert(newFileSource); session.commit(); } else { if (!newDataHash.equals(previous.getDataHash())) { - previous.setData(newData).setDataHash(newDataHash).setUpdatedAt(now); + previous.setData(newData).setDataHash(newDataHash).setUpdatedAt(now.getTime()); mapper.update(previous); session.commit(); } diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java index 17226a69a4c..8cb3ec22724 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java @@ -138,8 +138,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase { sourcePersister.persist(); FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidsame"); - assertThat(fileSourceDto.getCreatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-10T16:44:02+0200")); - assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now); + assertThat(fileSourceDto.getCreatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-10T16:44:02+0200").getTime()); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime()); assertThat(fileSourceDto.getData()).isEqualTo( ",,,,changed\r\n,,,,content\r\n"); assertThat(fileSourceDto.getDataHash()).isEqualTo("e41cca9c51ff853c748f708f39dfc035"); @@ -176,8 +176,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase { sourcePersister.persist(); FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidnew"); - assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now); - assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now); + assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now.getTime()); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime()); assertThat(fileSourceDto.getData()).isEqualTo( ",,,,foo\r\n,,,,bar\r\n,,,,biz\r\n"); assertThat(fileSourceDto.getDataHash()).isEqualTo("0c43ed6418d690ee0ffc3e43e6660967"); @@ -219,8 +219,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase { sourcePersister.persist(); FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidnew"); - assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now); - assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now); + assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now.getTime()); + assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime()); assertThat(fileSourceDto.getData()).isEqualTo( "123,julien,2014-10-11T16:44:02+0100,\"0,3,a\",foo\r\n" + "234,simon,2014-10-12T16:44:02+0100,\"0,1,cd\",bar\r\n" diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml index 972ba28b50f..fae5b462879 100644 --- a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml +++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml @@ -1,5 +1,5 @@ - + diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml index 972ba28b50f..fae5b462879 100644 --- a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml +++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml @@ -1,5 +1,5 @@ - + diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml index 77bf6e4d0c7..ac3c16563af 100644 --- a/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml +++ b/sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml @@ -1,6 +1,6 @@ - - + + diff --git a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDto.java b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDto.java index 3659ff20569..e992ee4a648 100644 --- a/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDto.java +++ b/sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDto.java @@ -21,14 +21,13 @@ package org.sonar.core.source.db; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import java.util.Date; public class FileSourceDto { private Long id; private String projectUuid; private String fileUuid; - private Date createdAt; - private Date updatedAt; + private long createdAt; + private long updatedAt; private String data; private String dataHash; @@ -78,20 +77,20 @@ public class FileSourceDto { return this; } - public Date getCreatedAt() { + public long getCreatedAt() { return createdAt; } - public FileSourceDto setCreatedAt(Date createdAt) { + public FileSourceDto setCreatedAt(long createdAt) { this.createdAt = createdAt; return this; } - public Date getUpdatedAt() { + public long getUpdatedAt() { return updatedAt; } - public FileSourceDto setUpdatedAt(Date updatedAt) { + public FileSourceDto setUpdatedAt(long updatedAt) { this.updatedAt = updatedAt; return this; } diff --git a/sonar-core/src/main/resources/org/sonar/core/persistence/schema-h2.ddl b/sonar-core/src/main/resources/org/sonar/core/persistence/schema-h2.ddl index 5842e9f4400..7de7f8d46c2 100644 --- a/sonar-core/src/main/resources/org/sonar/core/persistence/schema-h2.ddl +++ b/sonar-core/src/main/resources/org/sonar/core/persistence/schema-h2.ddl @@ -572,8 +572,8 @@ CREATE TABLE "FILE_SOURCES" ( "FILE_UUID" VARCHAR(50) NOT NULL, "DATA" CLOB(2147483647), "DATA_HASH" VARCHAR(50) NOT NULL, - "CREATED_AT" TIMESTAMP NOT NULL, - "UPDATED_AT" TIMESTAMP NOT NULL + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL ); -- ---------------------------------------------- diff --git a/sonar-core/src/test/java/org/sonar/core/source/db/FileSourceDaoTest.java b/sonar-core/src/test/java/org/sonar/core/source/db/FileSourceDaoTest.java index 04c3ec923b5..ddd45cedfa5 100644 --- a/sonar-core/src/test/java/org/sonar/core/source/db/FileSourceDaoTest.java +++ b/sonar-core/src/test/java/org/sonar/core/source/db/FileSourceDaoTest.java @@ -25,6 +25,8 @@ import org.junit.Test; import org.sonar.api.utils.DateUtils; import org.sonar.core.persistence.AbstractDaoTestCase; +import java.util.Date; + import static org.fest.assertions.Assertions.assertThat; public class FileSourceDaoTest extends AbstractDaoTestCase { @@ -45,15 +47,15 @@ public class FileSourceDaoTest extends AbstractDaoTestCase { assertThat(fileSourceDto.getDataHash()).isEqualTo("hash"); assertThat(fileSourceDto.getProjectUuid()).isEqualTo("abcd"); assertThat(fileSourceDto.getFileUuid()).isEqualTo("ab12"); - assertThat(fileSourceDto.getCreatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-29T16:44:02+0100")); - assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-30T16:44:02+0100")); + assertThat(new Date(fileSourceDto.getCreatedAt())).isEqualTo(DateUtils.parseDateTime("2014-10-29T16:44:02+0100")); + assertThat(new Date(fileSourceDto.getUpdatedAt())).isEqualTo(DateUtils.parseDateTime("2014-10-30T16:44:02+0100")); } @Test public void insert() throws Exception { dao.insert(new FileSourceDto().setProjectUuid("prj").setFileUuid("file").setData("bla bla").setDataHash("hash2") - .setCreatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100")) - .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100"))); + .setCreatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100").getTime()) + .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100").getTime())); checkTable("insert", "file_sources"); } @@ -61,7 +63,7 @@ public class FileSourceDaoTest extends AbstractDaoTestCase { @Test public void update() throws Exception { dao.update(new FileSourceDto().setId(101L).setProjectUuid("prj").setFileUuid("file").setData("updated data").setDataHash("hash2") - .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100"))); + .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100").getTime())); checkTable("update", "file_sources"); } diff --git a/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/insert-result.xml b/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/insert-result.xml index caa220868f4..c5d483b7e55 100644 --- a/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/insert-result.xml +++ b/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/insert-result.xml @@ -2,11 +2,11 @@ + created_at="1414597442000" updated_at="1414683842000" /> + created_at="1414770242000" updated_at="1414770242000" /> diff --git a/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/shared.xml b/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/shared.xml index 080676c67f2..fb3c258ddc0 100644 --- a/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/shared.xml +++ b/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/shared.xml @@ -2,6 +2,6 @@ + created_at="1414597442000" updated_at="1414683842000" /> diff --git a/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/update-result.xml b/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/update-result.xml index aee5ffcf405..465b2e52cb8 100644 --- a/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/update-result.xml +++ b/sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/update-result.xml @@ -2,7 +2,7 @@ + created_at="1414597442000" updated_at="1414770242000" /> -- 2.39.5