Revert "SONAR-11650 Drop DATA_TYPE from FILE_SOURCES"

This reverts commit 54ddc262

server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl

@@ -674,6 +674,7 @@ CREATE TABLE "FILE_SOURCES" (
   "LINE_HASHES_VERSION" INTEGER,
   "LINE_COUNT" INTEGER NOT NULL,
   "BINARY_DATA" BLOB,
+  "DATA_TYPE" VARCHAR(20),
   "DATA_HASH" VARCHAR(50),
   "SRC_HASH" VARCHAR(50),
   "REVISION" VARCHAR(100),
@@ -681,7 +682,7 @@ CREATE TABLE "FILE_SOURCES" (
   "UPDATED_AT" BIGINT NOT NULL
 );
 CREATE INDEX "FILE_SOURCES_PROJECT_UUID" ON "FILE_SOURCES" ("PROJECT_UUID");
-CREATE UNIQUE INDEX "FILE_SOURCES_FILE_UUID" ON "FILE_SOURCES" ("FILE_UUID");
+CREATE UNIQUE INDEX "FILE_SOURCES_UUID_TYPE" ON "FILE_SOURCES" ("FILE_UUID", "DATA_TYPE");
 CREATE INDEX "FILE_SOURCES_UPDATED_AT" ON "FILE_SOURCES" ("UPDATED_AT");
 
 

server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java

@@ -59,7 +59,7 @@ public class FileSourceDao implements Dao {
     PreparedStatement pstmt = null;
     ResultSet rs = null;
     try {
-      pstmt = connection.prepareStatement("SELECT line_hashes FROM file_sources WHERE file_uuid=?");
+      pstmt = connection.prepareStatement("SELECT line_hashes FROM file_sources WHERE file_uuid=? AND data_type='SOURCE'");
       pstmt.setString(1, fileUuid);
       rs = pstmt.executeQuery();
       if (rs.next()) {
@@ -93,7 +93,7 @@ public class FileSourceDao implements Dao {
     ResultSet rs = null;
     Reader reader = null;
     try {
-      pstmt = connection.prepareStatement("SELECT line_hashes FROM file_sources WHERE file_uuid=?");
+      pstmt = connection.prepareStatement("SELECT line_hashes FROM file_sources WHERE file_uuid=? AND data_type='SOURCE'");
       pstmt.setString(1, fileUuid);
       rs = pstmt.executeQuery();
       if (rs.next()) {

server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java

@@ -27,7 +27,7 @@ import org.apache.ibatis.session.ResultHandler;
 
 public interface FileSourceMapper {
 
-  List<FileSourceDto> selectHashesForProject(@Param("projectUuid") String projectUuid);
+  List<FileSourceDto> selectHashesForProject(@Param("projectUuid") String projectUuid, @Param("dataType") String dataType);
 
   @CheckForNull
   FileSourceDto selectByFileUuid(@Param("fileUuid") String fileUuid);

server/sonar-db-dao/src/main/protobuf/db-file-sources.proto

@@ -74,3 +74,24 @@ message Data {
   repeated Line lines = 1;
 }
 
+message Test {
+  optional string uuid = 1;
+  optional string name = 2;
+  optional TestStatus status = 3;
+  optional int64 execution_time_ms = 4;
+  optional string stacktrace = 5;
+  optional string msg = 6;
+  repeated CoveredFile covered_file = 7;
+
+  message CoveredFile {
+    optional string file_uuid = 1;
+    repeated int32 covered_line = 2 [packed = true];
+  }
+
+  enum TestStatus {
+    OK = 1;
+    FAILURE = 2;
+    ERROR = 3;
+    SKIPPED = 4;
+  }
+}

server/sonar-db-dao/src/main/resources/org/sonar/db/component/ComponentMapper.xml

@@ -226,6 +226,7 @@
       and p.scope='FIL'
     INNER JOIN file_sources fs ON
       fs.file_uuid=p.uuid
+      and fs.data_type='SOURCE'
     where
       root.uuid=#{projectUuid,jdbcType=VARCHAR}
   </select>
@@ -240,6 +241,7 @@
     FROM projects p
     INNER JOIN file_sources fs ON
       fs.file_uuid=p.uuid
+      and fs.data_type='SOURCE'
     <include refid="modulesTreeQuery"/>
   </select>
 
@@ -588,6 +590,7 @@
     from projects p
     inner join file_sources fs on
       fs.file_uuid = p.uuid
+      and fs.data_type = 'SOURCE'
     where
       p.project_uuid = #{projectUuid,jdbcType=VARCHAR}
       and p.enabled = ${_true}

server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml

@@ -22,6 +22,7 @@
       file_sources
     where
       file_uuid = #{fileUuid,jdbcType=VARCHAR}
+      and data_type = 'SOURCE'
   </select>
 
   <select id="selectHashesForProject" parameterType="map" resultType="org.sonar.db.source.FileSourceDto">
@@ -32,10 +33,11 @@
       src_hash as srcHash,
       revision,
       updated_at as updatedAt
-    from
+      from
       file_sources
     where
       project_uuid = #{projectUuid,jdbcType=VARCHAR}
+      and data_type = 'SOURCE'
   </select>
 
   <select id="scrollLineHashes" parameterType="map" resultType="org.sonar.db.source.LineHashesWithUuidDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
@@ -46,6 +48,7 @@
     from projects p
     inner join file_sources fs on
       fs.file_uuid = p.uuid
+      and fs.data_type = 'SOURCE'
     where
       p.uuid in
       <foreach collection="fileUuids" item="fileUuid" open="(" close=")" separator=",">
@@ -61,6 +64,7 @@
       file_sources
     WHERE
       file_uuid = #{fileUuid,jdbcType=VARCHAR}
+      and data_type = 'SOURCE'
   </select>
 
   <insert id="insert" parameterType="org.sonar.db.source.FileSourceDto" useGeneratedKeys="false">
@@ -76,6 +80,7 @@
       line_count,
       data_hash,
       src_hash,
+      data_type,
       revision
     )
     values
@@ -90,6 +95,7 @@
       #{lineCount,jdbcType=INTEGER},
       #{dataHash,jdbcType=VARCHAR},
       #{srcHash,jdbcType=VARCHAR},
+      'SOURCE',
       #{revision,jdbcType=VARCHAR}
     )
   </insert>

server/sonar-db-dao/src/test/java/org/sonar/db/source/FileSourceDaoTest.java

@@ -96,6 +96,16 @@ public class FileSourceDaoTest {
     assertThat(fn.result).isNull();
   }
 
+  @Test
+  public void no_line_hashes_when_only_test_data() {
+    dbTester.prepareDbUnit(getClass(), "no_line_hashes_when_only_test_data.xml");
+
+    ReaderToStringConsumer fn = new ReaderToStringConsumer();
+    underTest.readLineHashesStream(dbSession, "FILE1_UUID", fn);
+
+    assertThat(fn.result).isNull();
+  }
+
   @Test
   public void insert() {
     FileSourceDto expected = new FileSourceDto()

server/sonar-db-dao/src/test/resources/org/sonar/db/purge/PurgeDaoTest/shouldDeleteProject.xml

@@ -224,5 +224,6 @@
                 line_count="0"
                 data_hash="321654987"
                 created_at="123456789"
-                updated_at="123456789"/>
+                updated_at="123456789"
+                data_type="SOURCE"/>
 </dataset>

server/sonar-db-dao/src/test/resources/org/sonar/db/source/FileSourceDaoTest/no_line_hashes_when_only_test_data.xml

@@ -0,0 +1,11 @@
+<dataset>
+
+    <file_sources id="101" project_uuid="PRJ_UUID" file_uuid="FILE1_UUID"
+                  binary_data="abcde" data_hash="[null]"
+                  line_hashes="[null]"
+                  line_count="0"
+                  src_hash="[null]"
+                  line_hashes_version="[null]"
+                  created_at="1500000000000" updated_at="1500000000000"  data_type="TEST" />
+
+</dataset>

server/sonar-db-dao/src/test/resources/org/sonar/db/source/FileSourceDaoTest/shared.xml

@@ -5,7 +5,7 @@
                   line_hashes="ABC\nDEF\nGHI"
                   line_count="3"
                   src_hash="FILE_HASH" revision="123456789"
-                  created_at="1500000000000" updated_at="1500000000000"
+                  created_at="1500000000000" updated_at="1500000000000"  data_type="SOURCE"
                   line_hashes_version="[null]"/>
 
 </dataset>

server/sonar-db-dao/src/test/resources/org/sonar/db/source/FileSourceDaoTest/update-result.xml

@@ -6,7 +6,7 @@
                 line_hashes="NEW_LINE_HASHES"
                 line_count="1"
                 src_hash="NEW_FILE_HASH" revision="987654321"
-                created_at="1500000000000" updated_at="1500000000002"
+                created_at="1500000000000" updated_at="1500000000002"  data_type="SOURCE"
                 line_hashes_version="1" />
 
 

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/step/DdlChange.java

@@ -24,7 +24,6 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.List;
 import java.util.regex.Pattern;
-import org.sonar.api.config.Configuration;
 import org.sonar.db.Database;
 import org.sonar.db.dialect.Dialect;
 
@@ -55,10 +54,6 @@ public abstract class DdlChange implements MigrationStep {
 
   public abstract void execute(Context context) throws SQLException;
 
-  protected static boolean isSonarCloud(Configuration configuration) {
-    return configuration.getBoolean("sonar.sonarcloud.enabled").orElse(false);
-  }
-
   protected Database getDatabase() {
     return db;
   }

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v77/DbVersion77.java

@@ -30,7 +30,6 @@ public class DbVersion77 implements DbVersion {
       .add(2600, "Drop elasticsearch index 'tests'", DropElasticsearchIndexTests.class)
       .add(2601, "Delete lines with DATA_TYPE='TEST' from table FILES_SOURCE", DeleteTestDataTypeFromFileSources.class)
       .add(2602, "Add column LAST_CONNECTION_DATE to USERS table", AddLastConnectionDateToUsers.class)
-      .add(2603, "Add column LAST_USED_DATE to USER_TOKENS table", AddLastConnectionDateToUserTokens.class)
-      .add(2604, "Drop DATA_TYPE column from FILE_SOURCES table", DropDataTypeFromFileSources.class);
+      .add(2603, "Add column LAST_USED_DATE to USER_TOKENS table", AddLastConnectionDateToUserTokens.class);
   }
 }

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v77/DropDataTypeFromFileSources.java

@@ -1,63 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.platform.db.migration.version.v77;
-
-import java.sql.SQLException;
-import org.sonar.db.Database;
-import org.sonar.server.platform.db.migration.SupportsBlueGreen;
-import org.sonar.server.platform.db.migration.sql.CreateIndexBuilder;
-import org.sonar.server.platform.db.migration.sql.DropColumnsBuilder;
-import org.sonar.server.platform.db.migration.sql.DropIndexBuilder;
-import org.sonar.server.platform.db.migration.step.DdlChange;
-
-import static org.sonar.server.platform.db.migration.def.VarcharColumnDef.newVarcharColumnDefBuilder;
-
-@SupportsBlueGreen
-public class DropDataTypeFromFileSources extends DdlChange {
-
-  private static final String TABLE_NAME = "file_sources";
-
-  public DropDataTypeFromFileSources(Database db) {
-    super(db);
-  }
-
-  @Override
-  public void execute(Context context) throws SQLException {
-    context.execute(new DropIndexBuilder(getDialect())
-      .setTable(TABLE_NAME)
-      .setName("file_sources_uuid_type")
-      .build());
-
-    context.execute(new DropColumnsBuilder(getDialect(), TABLE_NAME, "data_type")
-      .build());
-
-    context.execute(new CreateIndexBuilder(getDialect())
-      .setTable(TABLE_NAME)
-      .setName("file_sources_file_uuid")
-      .setUnique(true)
-      .addColumn(newVarcharColumnDefBuilder()
-        .setColumnName("file_uuid")
-        .setIsNullable(false)
-        .setLimit(50)
-        .build())
-      .build());
-  }
-
-}

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v77/DbVersion77Test.java

@@ -36,7 +36,7 @@ public class DbVersion77Test {
 
   @Test
   public void verify_migration_count() {
-    verifyMigrationCount(underTest, 5);
+    verifyMigrationCount(underTest, 4);
   }
 
 }

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v77/DropDataTypeFromFileSourcesTest.java

@@ -1,57 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-package org.sonar.server.platform.db.migration.version.v77;
-
-import java.sql.SQLException;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.sonar.db.CoreDbTester;
-
-public class DropDataTypeFromFileSourcesTest {
-
-  private static final String TABLE = "file_sources";
-
-  @Rule
-  public final CoreDbTester db = CoreDbTester.createForSchema(DropDataTypeFromFileSourcesTest.class, "file_sources.sql");
-  @Rule
-  public ExpectedException expectedException = ExpectedException.none();
-
-  private DropDataTypeFromFileSources underTest = new DropDataTypeFromFileSources(db.database());
-
-  @Test
-  public void drop_column_and_recreate_index() throws SQLException {
-    underTest.execute();
-
-    db.assertColumnDoesNotExist(TABLE, "data_type");
-    db.assertUniqueIndex(TABLE, "file_sources_file_uuid", "file_uuid");
-  }
-
-  @Test
-  public void migration_is_not_re_entrant() throws SQLException {
-    underTest.execute();
-
-    expectedException.expect(IllegalStateException.class);
-
-    underTest.execute();
-  }
-
-}

server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v77/DropDataTypeFromFileSourcesTest/file_sources.sql

@@ -1,18 +0,0 @@
-CREATE TABLE "FILE_SOURCES" (
-  "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1),
-  "PROJECT_UUID" VARCHAR(50) NOT NULL,
-  "FILE_UUID" VARCHAR(50) NOT NULL,
-  "LINE_HASHES" CLOB,
-  "LINE_HASHES_VERSION" INTEGER,
-  "LINE_COUNT" INTEGER NOT NULL,
-  "BINARY_DATA" BLOB,
-  "DATA_TYPE" VARCHAR(20),
-  "DATA_HASH" VARCHAR(50),
-  "SRC_HASH" VARCHAR(50),
-  "REVISION" VARCHAR(100),
-  "CREATED_AT" BIGINT NOT NULL,
-  "UPDATED_AT" BIGINT NOT NULL
-);
-CREATE INDEX "FILE_SOURCES_PROJECT_UUID" ON "FILE_SOURCES" ("PROJECT_UUID");
-CREATE UNIQUE INDEX "FILE_SOURCES_UUID_TYPE" ON "FILE_SOURCES" ("FILE_UUID", "DATA_TYPE");
-CREATE INDEX "FILE_SOURCES_UPDATED_AT" ON "FILE_SOURCES" ("UPDATED_AT");