]> source.dussan.org Git - sonarqube.git/commitdiff
SONAR-5826 Use (big) integer column instead of DB timestamp for file_sources
authorJean-Baptiste Lievremont <jean-baptiste.lievremont@sonarsource.com>
Mon, 24 Nov 2014 16:12:25 +0000 (17:12 +0100)
committerJean-Baptiste Lievremont <jean-baptiste.lievremont@sonarsource.com>
Mon, 24 Nov 2014 17:16:12 +0000 (18:16 +0100)
21 files changed:
server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java
server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java
server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java
server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after-with-scm.xml
server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/after.xml
server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesMigrationTest/schema.sql
server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/schema.sql
server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineResultSetIteratorTest/source-with-scm.xml
server/sonar-web/src/main/webapp/WEB-INF/db/migrate/713_create_file_sources.rb
sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/file_sources.xml
sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistDontTouchUnchanged-result.xml
sonar-batch/src/test/resources/org/sonar/batch/index/SourcePersisterTest/testPersistEmptyFile-result.xml
sonar-core/src/main/java/org/sonar/core/source/db/FileSourceDto.java
sonar-core/src/main/resources/org/sonar/core/persistence/schema-h2.ddl
sonar-core/src/test/java/org/sonar/core/source/db/FileSourceDaoTest.java
sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/insert-result.xml
sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/shared.xml
sonar-core/src/test/resources/org/sonar/core/source/db/FileSourceDaoTest/update-result.xml

index 515a6fde99e440c44a7ab959632e6bf6b33a8843..3c09e05947f427223698962e1f4db2ae81c2c6da 100644 (file)
@@ -60,8 +60,8 @@ public class FeedFileSources extends BaseDataChange {
 
       update.setString(1, projectUuid)
         .setString(2, fileUuid)
-        .setDate(3, now)
-        .setDate(4, updatedAt == null ? now : updatedAt)
+        .setLong(3, now.getTime())
+        .setLong(4, (updatedAt == null ? now : updatedAt).getTime())
         .setString(5, sourceData)
         .setString(6, "");
 
index 2895663d2cc3bc8ec81223be372818744f7b04e3..52f758df34c175a843d227758ff3e0862fa5c411 100644 (file)
@@ -92,6 +92,9 @@ public class SourceLineIndexer implements ServerComponent {
       // use cache. Will not work with Tomcat cluster.
       result = lastUpdatedAt;
     }
+    System.out.println("\n\n"
+      + "   Checking for source lines > " + result
+      + "\n\n");
     return result;
   }
 
index a89a0ae534ac4883bab60315ba747606689fa69a..86a22f2dc3631fc4619b41a199ab5e41b174c6c7 100644 (file)
@@ -37,7 +37,6 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Timestamp;
 import java.util.Collection;
 import java.util.Date;
 import java.util.List;
@@ -60,15 +59,16 @@ class SourceLineResultSetIterator extends ResultSetIterator<Collection<SourceLin
 
   private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from file_sources";
 
-  private static final String SQL_AFTER_DATE = SQL_ALL + " where updated_at>=?";
+  private static final String SQL_AFTER_DATE = SQL_ALL + " where updated_at>?";
 
   static SourceLineResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) {
     try {
       String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL;
       PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql);
       if (afterDate > 0L) {
-        stmt.setTimestamp(1, new Timestamp(afterDate));
+        stmt.setLong(1, afterDate);
       }
+      System.out.println(sql);
       return new SourceLineResultSetIterator(stmt);
     } catch (SQLException e) {
       throw new IllegalStateException("Fail to prepare SQL request to select all file sources", e);
@@ -83,7 +83,7 @@ class SourceLineResultSetIterator extends ResultSetIterator<Collection<SourceLin
   protected Collection<SourceLineDoc> read(ResultSet rs) throws SQLException {
     String projectUuid = rs.getString(1);
     String fileUuid = rs.getString(2);
-    Date updatedAt = SqlUtil.getDate(rs, 4);
+    Date updatedAt = new Date(SqlUtil.getLong(rs, 4));
 
     int line = 1;
     List<SourceLineDoc> lines = Lists.newArrayList();
index e58ad44f63d5c1e7286b95d86912dc6e29b88ada..7ca40e26238e42319e0aa34a28293b6858a7bdbe 100644 (file)
@@ -87,7 +87,7 @@ public class SourceLineResultSetIteratorTest {
     db.prepareDbUnit(getClass(), "source-with-scm.xml");
 
     SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, db.openConnection(),
-      DateUtils.parseDateTime("2014-11-01T16:44:02+0100").getTime());
+      DateUtils.parseDateTime("2014-11-17T16:44:02+0100").getTime());
     assertThat(iterator.hasNext()).isFalse();
   }
 
index 6d17e4011798ef67bfb21efc48df2a727f981550..c3948bde3b1e18b3bd92fab465f6ee519c72ad55 100644 (file)
@@ -1,6 +1,6 @@
 <dataset>
 
-  <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="2014-11-17 16:27:00.000" updated_at="2014-10-31 16:44:02.000"
+  <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="1416238020000" updated_at="1414770242000"
     data="aef12a,alice,2014-04-25T12:34:56+0100,,class Foo {&#13;&#10;abe465,bob,2014-07-25T12:34:56+0100,,  // Empty&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,,}&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,,&#13;&#10;" data_hash="" />
 
 </dataset>
index 5369e25ab2d4c24d515878f6e0cd0c4a96b79b90..b5de4a9cbfe4f4ba3bacc6b8ce56a9cfe041bd3d 100644 (file)
@@ -1,6 +1,6 @@
 <dataset>
 
-  <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="2014-11-17 16:27:00.000" updated_at="2014-10-31 16:44:02.000"
+  <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="1416238020000" updated_at="1414770242000"
     data=",,,,class Foo {&#13;&#10;,,,,  // Empty&#13;&#10;,,,,}&#13;&#10;,,,,&#13;&#10;" data_hash="" />
 
 </dataset>
index 0a99ce5b1d08259aed050e5e519d5d3bd8b0df6a..658181774d5b78640d3f31bf09213ac1d69c9896 100644 (file)
@@ -113,6 +113,6 @@ CREATE TABLE "FILE_SOURCES" (
   "FILE_UUID" VARCHAR(50) NOT NULL,
   "DATA" CLOB(2147483647),
   "DATA_HASH" VARCHAR(50) NOT NULL,
-  "CREATED_AT" TIMESTAMP NOT NULL,
-  "UPDATED_AT" TIMESTAMP NOT NULL
+  "CREATED_AT" BIGINT NOT NULL,
+  "UPDATED_AT" BIGINT NOT NULL
 );
index d045f1f012b7d1ddcf801f702d4cc19c5a36d67c..3032e93f25636cae6aac37abd2c7f28a072bc30b 100644 (file)
@@ -5,6 +5,6 @@ CREATE TABLE "FILE_SOURCES" (
   "FILE_UUID" VARCHAR(50) NOT NULL,
   "DATA" CLOB(2147483647),
   "DATA_HASH" VARCHAR(50) NOT NULL,
-  "CREATED_AT" TIMESTAMP NOT NULL,
-  "UPDATED_AT" TIMESTAMP NOT NULL
+  "CREATED_AT" BIGINT NOT NULL,
+  "UPDATED_AT" BIGINT NOT NULL
 );
index 3f2161acff0bb1bc2f7c13997b4923ab85b443d6..f1498ba9c5600d79eaeb558a89cb4629e6d05ca8 100644 (file)
@@ -1,6 +1,6 @@
 <dataset>
 
-  <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="2014-11-17 16:27:00.000" updated_at="2014-10-31 16:44:02.000"
+  <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="1416238020000" updated_at="1416239042000"
     data="" data_hash="" />
 
 </dataset>
index d8cdc1b27760dd0006821cbba5f4cc7cd51d4d62..422f355f95b7e47d3d4b22cf652c58e455073c37 100644 (file)
@@ -29,8 +29,8 @@ class CreateFileSources < ActiveRecord::Migration
       t.column :file_uuid,    :string,   :limit => 50, :null => false
       t.column :data,         :text,     :null => true
       t.column :data_hash,    :string,   :limit => 50, :null => true
-      t.column :created_at,   :datetime, :null => false
-      t.column :updated_at,   :datetime, :null => false
+      t.column :created_at,   :integer,  :limit => 8, :null => false 
+      t.column :updated_at,   :integer,  :limit => 8, :null => false
     end
 
     if dialect()=='mysql'
index b0f1199117eb08d703fa961310e3bc1d9f777052..6e42a07c18ecc06b705e5c2d25d5d3036e38eab8 100644 (file)
@@ -56,12 +56,7 @@ import javax.annotation.Nullable;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import static com.google.common.base.Charsets.UTF_8;
 
@@ -154,13 +149,13 @@ public class SourcePersister implements ScanPersister {
     if (previous == null) {
       FileSourceDto newFileSource = new FileSourceDto().setProjectUuid(projectTree.getRootProject().getUuid()).setFileUuid(fileUuid).setData(newData)
         .setDataHash(newDataHash)
-        .setCreatedAt(now)
-        .setUpdatedAt(now);
+        .setCreatedAt(now.getTime())
+        .setUpdatedAt(now.getTime());
       mapper.insert(newFileSource);
       session.commit();
     } else {
       if (!newDataHash.equals(previous.getDataHash())) {
-        previous.setData(newData).setDataHash(newDataHash).setUpdatedAt(now);
+        previous.setData(newData).setDataHash(newDataHash).setUpdatedAt(now.getTime());
         mapper.update(previous);
         session.commit();
       }
index 17226a69a4c384462e7c08e3d3f0fdb37418c507..8cb3ec22724751f2fbf59b021451aba5964a9188 100644 (file)
@@ -138,8 +138,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     sourcePersister.persist();
 
     FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidsame");
-    assertThat(fileSourceDto.getCreatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-10T16:44:02+0200"));
-    assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now);
+    assertThat(fileSourceDto.getCreatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-10T16:44:02+0200").getTime());
+    assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getData()).isEqualTo(
       ",,,,changed\r\n,,,,content\r\n");
     assertThat(fileSourceDto.getDataHash()).isEqualTo("e41cca9c51ff853c748f708f39dfc035");
@@ -176,8 +176,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
 
     sourcePersister.persist();
     FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidnew");
-    assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now);
-    assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now);
+    assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now.getTime());
+    assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getData()).isEqualTo(
       ",,,,foo\r\n,,,,bar\r\n,,,,biz\r\n");
     assertThat(fileSourceDto.getDataHash()).isEqualTo("0c43ed6418d690ee0ffc3e43e6660967");
@@ -219,8 +219,8 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     sourcePersister.persist();
 
     FileSourceDto fileSourceDto = new FileSourceDao(getMyBatis()).select("uuidnew");
-    assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now);
-    assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now);
+    assertThat(fileSourceDto.getCreatedAt()).isEqualTo(now.getTime());
+    assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(now.getTime());
     assertThat(fileSourceDto.getData()).isEqualTo(
       "123,julien,2014-10-11T16:44:02+0100,\"0,3,a\",foo\r\n"
         + "234,simon,2014-10-12T16:44:02+0100,\"0,1,cd\",bar\r\n"
index 972ba28b50f0c81149a6d6adf3b67b24028bbffb..fae5b462879f18f7a25c4a29228710e601dc7c41 100644 (file)
@@ -1,5 +1,5 @@
 <dataset>
 
-  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
+  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="1412952242000" updated_at="1412952242000" />
 
 </dataset>
index 972ba28b50f0c81149a6d6adf3b67b24028bbffb..fae5b462879f18f7a25c4a29228710e601dc7c41 100644 (file)
@@ -1,5 +1,5 @@
 <dataset>
 
-  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
+  <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="1412952242000" updated_at="1412952242000" />
 
 </dataset>
index 77bf6e4d0c74132e6f334c8619286e517b787469..ac3c16563af5be1f2034a934502809dbe538c7fc 100644 (file)
@@ -1,6 +1,6 @@
 <dataset>
-    <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="2014-10-10 16:44:02.000" updated_at="2014-10-10 16:44:02.000" />
-    <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidempty" data="[null]" data_hash="0" created_at="2014-10-29 16:44:02.000" updated_at="2014-10-29 16:44:02.000" />
+    <file_sources id="101" project_uuid="projectUuid" file_uuid="uuidsame" data=",,,,unchanged&#13;&#10;,,,,content&#13;&#10;" data_hash="ee716d4ed9faae16eb9167714442a3bc" created_at="1412952242000" updated_at="1412952242000" />
+    <file_sources id="102" project_uuid="projectUuid" file_uuid="uuidempty" data="[null]" data_hash="0" created_at="1414597442000" updated_at="1414597442000" />
 </dataset>
 
 
index 3659ff2056946a5f9ec402f484d57c28003212ef..e992ee4a648b2de175da5c85356844fa0649e4d9 100644 (file)
@@ -21,14 +21,13 @@ package org.sonar.core.source.db;
 
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
-import java.util.Date;
 
 public class FileSourceDto {
   private Long id;
   private String projectUuid;
   private String fileUuid;
-  private Date createdAt;
-  private Date updatedAt;
+  private long createdAt;
+  private long updatedAt;
   private String data;
   private String dataHash;
 
@@ -78,20 +77,20 @@ public class FileSourceDto {
     return this;
   }
 
-  public Date getCreatedAt() {
+  public long getCreatedAt() {
     return createdAt;
   }
 
-  public FileSourceDto setCreatedAt(Date createdAt) {
+  public FileSourceDto setCreatedAt(long createdAt) {
     this.createdAt = createdAt;
     return this;
   }
 
-  public Date getUpdatedAt() {
+  public long getUpdatedAt() {
     return updatedAt;
   }
 
-  public FileSourceDto setUpdatedAt(Date updatedAt) {
+  public FileSourceDto setUpdatedAt(long updatedAt) {
     this.updatedAt = updatedAt;
     return this;
   }
index 5842e9f4400b77896713bb428bdccc778ff85179..7de7f8d46c20d61acbdb61a512750b1976cbb2b4 100644 (file)
@@ -572,8 +572,8 @@ CREATE TABLE "FILE_SOURCES" (
   "FILE_UUID" VARCHAR(50) NOT NULL,
   "DATA" CLOB(2147483647),
   "DATA_HASH" VARCHAR(50) NOT NULL,
-  "CREATED_AT" TIMESTAMP NOT NULL,
-  "UPDATED_AT" TIMESTAMP NOT NULL
+  "CREATED_AT" BIGINT NOT NULL,
+  "UPDATED_AT" BIGINT NOT NULL
 );
 
 -- ----------------------------------------------
index 04c3ec923b500a455f3cd71d51e15d8e8d3e8cf0..ddd45cedfa5352645cebfa6ad695a12498579b1b 100644 (file)
@@ -25,6 +25,8 @@ import org.junit.Test;
 import org.sonar.api.utils.DateUtils;
 import org.sonar.core.persistence.AbstractDaoTestCase;
 
+import java.util.Date;
+
 import static org.fest.assertions.Assertions.assertThat;
 
 public class FileSourceDaoTest extends AbstractDaoTestCase {
@@ -45,15 +47,15 @@ public class FileSourceDaoTest extends AbstractDaoTestCase {
     assertThat(fileSourceDto.getDataHash()).isEqualTo("hash");
     assertThat(fileSourceDto.getProjectUuid()).isEqualTo("abcd");
     assertThat(fileSourceDto.getFileUuid()).isEqualTo("ab12");
-    assertThat(fileSourceDto.getCreatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
-    assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(DateUtils.parseDateTime("2014-10-30T16:44:02+0100"));
+    assertThat(new Date(fileSourceDto.getCreatedAt())).isEqualTo(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
+    assertThat(new Date(fileSourceDto.getUpdatedAt())).isEqualTo(DateUtils.parseDateTime("2014-10-30T16:44:02+0100"));
   }
 
   @Test
   public void insert() throws Exception {
     dao.insert(new FileSourceDto().setProjectUuid("prj").setFileUuid("file").setData("bla bla").setDataHash("hash2")
-      .setCreatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100"))
-      .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100")));
+      .setCreatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100").getTime())
+      .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100").getTime()));
 
     checkTable("insert", "file_sources");
   }
@@ -61,7 +63,7 @@ public class FileSourceDaoTest extends AbstractDaoTestCase {
   @Test
   public void update() throws Exception {
     dao.update(new FileSourceDto().setId(101L).setProjectUuid("prj").setFileUuid("file").setData("updated data").setDataHash("hash2")
-      .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100")));
+      .setUpdatedAt(DateUtils.parseDateTime("2014-10-31T16:44:02+0100").getTime()));
 
     checkTable("update", "file_sources");
   }
index caa220868f4fc48a4560370c5d119e150871bdee..c5d483b7e55f9138cd5b87485b8fe2280e0e61f2 100644 (file)
@@ -2,11 +2,11 @@
 
     <file_sources id="101" project_uuid="abcd" file_uuid="ab12"
                   data="aef12a,alice,2014-04-25T12:34:56+0100,,class Foo" data_hash="hash"
-                  created_at="2014-10-29 16:44:02.000" updated_at="2014-10-30 16:44:02.000" />
+                  created_at="1414597442000" updated_at="1414683842000" />
 
 
     <file_sources id="102" project_uuid="prj" file_uuid="file"
                   data="bla bla" data_hash="hash2"
-                  created_at="2014-10-31 16:44:02.000" updated_at="2014-10-31 16:44:02.000" />
+                  created_at="1414770242000" updated_at="1414770242000" />
 
 </dataset>
index 080676c67f2a38c575f357c59850faf3f1f3a948..fb3c258ddc00727aacfc442353e0023b8e97c366 100644 (file)
@@ -2,6 +2,6 @@
 
     <file_sources id="101" project_uuid="abcd" file_uuid="ab12"
                   data="aef12a,alice,2014-04-25T12:34:56+0100,,class Foo" data_hash="hash"
-                  created_at="2014-10-29 16:44:02.000" updated_at="2014-10-30 16:44:02.000" />
+                  created_at="1414597442000" updated_at="1414683842000" />
 
 </dataset>
index aee5ffcf405cfb37c3e991ee58316efec6e26c2b..465b2e52cb8f4475a45efd126e1a298bdbf21b10 100644 (file)
@@ -2,7 +2,7 @@
 
     <file_sources id="101" project_uuid="abcd" file_uuid="ab12"
                   data="updated data" data_hash="hash2"
-                  created_at="2014-10-29 16:44:02.000" updated_at="2014-10-31 16:44:02.000" />
+                  created_at="1414597442000" updated_at="1414770242000" />
 
 
 </dataset>