From 37c3989875a4407c1e00128272a6f96800d5fb43 Mon Sep 17 00:00:00 2001 From: Janos Gyerik Date: Thu, 9 May 2019 15:24:18 +0200 Subject: [PATCH] SONAR-12061 Recalculate out-of-date portfolios at scheduled hours --- .../java/org/sonar/db/version/SqTables.java | 1 + .../InternalComponentPropertiesDao.java | 29 ++- .../InternalComponentPropertiesMapper.java | 8 + .../InternalComponentPropertiesMapper.xml | 44 +++- .../InternalComponentPropertiesDaoTest.java | 203 ++++++++---------- 5 files changed, 158 insertions(+), 127 deletions(-) diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java index bbe0c8f6e31..0da5a55c381 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java @@ -70,6 +70,7 @@ public final class SqTables { "groups", "groups_users", "group_roles", + "internal_component_props", "internal_properties", "issues", "issue_changes", diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesDao.java index b9ae5bfa306..842e1f270fa 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesDao.java @@ -20,11 +20,15 @@ package org.sonar.db.property; import java.util.Optional; +import java.util.Set; import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; import org.sonar.db.Dao; import org.sonar.db.DbSession; +/** + * A simple key-value store per component. + */ public class InternalComponentPropertiesDao implements Dao { private final System2 system2; @@ -35,7 +39,7 @@ public class InternalComponentPropertiesDao implements Dao { this.uuidFactory = uuidFactory; } - public void insertOrUpdate(DbSession dbSession, InternalComponentPropertyDto dto) { + private void insertOrUpdate(DbSession dbSession, InternalComponentPropertyDto dto) { InternalComponentPropertiesMapper mapper = getMapper(dbSession); dto.setUpdatedAt(system2.now()); @@ -49,6 +53,22 @@ public class InternalComponentPropertiesDao implements Dao { mapper.insert(dto); } + /** + * For the given component uuid, update the value of the specified key, if exists, + * otherwise insert it. + */ + public void insertOrUpdate(DbSession dbSession, String componentUuid, String key, String value) { + insertOrUpdate(dbSession, new InternalComponentPropertyDto().setComponentUuid(componentUuid).setKey(key).setValue(value)); + } + + /** + * For the given component uuid, replace the value of the specified key "atomically": + * only replace if the old value is still the same as the current value. + */ + public void replaceValue(DbSession dbSession, String componentUuid, String key, String oldValue, String newValue) { + getMapper(dbSession).replaceValue(componentUuid, key, oldValue, newValue, system2.now()); + } + public Optional selectByComponentUuidAndKey(DbSession dbSession, String componentUuid, String key) { return getMapper(dbSession).selectByComponentUuidAndKey(componentUuid, key); } @@ -57,6 +77,13 @@ public class InternalComponentPropertiesDao implements Dao { return getMapper(dbSession).deleteByComponentUuidAndKey(componentUuid, key); } + /** + * Select the projects.kee values for internal component properties having specified key and value. + */ + public Set selectDbKeys(DbSession dbSession, String key, String value) { + return getMapper(dbSession).selectDbKeys(key, value); + } + private static InternalComponentPropertiesMapper getMapper(DbSession dbSession) { return dbSession.getMapper(InternalComponentPropertiesMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesMapper.java index 3ddc22c6634..9daa9c641f8 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/property/InternalComponentPropertiesMapper.java @@ -20,6 +20,7 @@ package org.sonar.db.property; import java.util.Optional; +import java.util.Set; import org.apache.ibatis.annotations.Param; public interface InternalComponentPropertiesMapper { @@ -30,6 +31,13 @@ public interface InternalComponentPropertiesMapper { int update(@Param("dto") InternalComponentPropertyDto dto); + /** + * Replace value (and update updated_at) only if current value matches oldValue + */ + void replaceValue(@Param("componentUuid") String componentUuid, @Param("key") String key, @Param("oldValue") String oldValue, @Param("newValue") String newValue, + @Param("updatedAt") Long updatedAt); + int deleteByComponentUuidAndKey(@Param("componentUuid") String componentUuid, @Param("key") String key); + Set selectDbKeys(@Param("key") String key, @Param("value") String value); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/property/InternalComponentPropertiesMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/property/InternalComponentPropertiesMapper.xml index 1ad615315e7..ec18d5afa4c 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/property/InternalComponentPropertiesMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/property/InternalComponentPropertiesMapper.xml @@ -19,6 +19,21 @@ + + insert into internal_component_props ( @@ -30,12 +45,12 @@ created_at ) values ( - #{dto.uuid}, - #{dto.componentUuid}, - #{dto.key}, - #{dto.value}, - #{dto.updatedAt}, - #{dto.createdAt} + #{dto.uuid, jdbcType=VARCHAR}, + #{dto.componentUuid, jdbcType=VARCHAR}, + #{dto.key, jdbcType=VARCHAR}, + #{dto.value, jdbcType=VARCHAR}, + #{dto.updatedAt, jdbcType=BIGINT}, + #{dto.createdAt, jdbcType=BIGINT} ) @@ -47,7 +62,20 @@ component_uuid = #{dto.componentUuid, jdbcType=VARCHAR} - AND kee = #{dto.key} + AND kee = #{dto.key, jdbcType=VARCHAR} + + + + + update internal_component_props + + value = #{newValue, jdbcType=VARCHAR}, + updated_at = #{updatedAt, jdbcType=BIGINT} + + + component_uuid = #{componentUuid, jdbcType=VARCHAR} + AND kee = #{key, jdbcType=VARCHAR} + AND value = #{oldValue, jdbcType=VARCHAR} @@ -55,7 +83,7 @@ DELETE FROM internal_component_props component_uuid = #{componentUuid, jdbcType=VARCHAR} - AND kee = #{key,jdbcType=VARCHAR} + AND kee = #{key, jdbcType=VARCHAR} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/property/InternalComponentPropertiesDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/property/InternalComponentPropertiesDaoTest.java index b7eecb8b1dc..317492888ac 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/property/InternalComponentPropertiesDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/property/InternalComponentPropertiesDaoTest.java @@ -19,11 +19,7 @@ */ package org.sonar.db.property; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import org.apache.commons.lang.math.RandomUtils; -import org.assertj.core.api.AbstractAssert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -32,6 +28,8 @@ import org.sonar.core.util.UuidFactory; import org.sonar.core.util.UuidFactoryFast; import org.sonar.db.DbSession; import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.organization.OrganizationDto; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -55,42 +53,83 @@ public class InternalComponentPropertiesDaoTest { @Test public void insertOrUpdate_insert_property_if_it_doesnt_already_exist() { - InternalComponentPropertyDto dto = new InternalComponentPropertyDto() - .setKey(SOME_KEY) - .setComponentUuid(SOME_COMPONENT) - .setValue(SOME_VALUE); - - long now = RandomUtils.nextLong(); - when(system2.now()).thenReturn(now); - - underTest.insertOrUpdate(dbSession, dto); - - assertThatInternalProperty(dto.getUuid()) - .hasComponentUuid(SOME_COMPONENT) - .hasKey(SOME_KEY) - .hasValue(SOME_VALUE) - .hasUpdatedAt(now) - .hasCreatedAt(now); + long createdAt = 10L; + when(system2.now()).thenReturn(createdAt); + + underTest.insertOrUpdate(dbSession, SOME_COMPONENT, SOME_KEY, SOME_VALUE); + + InternalComponentPropertyDto dto = underTest.selectByComponentUuidAndKey(dbSession, SOME_COMPONENT, SOME_KEY).get(); + + assertThat(dto.getComponentUuid()).isEqualTo(SOME_COMPONENT); + assertThat(dto.getKey()).isEqualTo(SOME_KEY); + assertThat(dto.getValue()).isEqualTo(SOME_VALUE); + assertThat(dto.getUpdatedAt()).isEqualTo(createdAt); + assertThat(dto.getCreatedAt()).isEqualTo(createdAt); } @Test public void insertOrUpdate_update_property_if_it_already_exists() { - long creationDate = 10L; - when(system2.now()).thenReturn(creationDate); + long createdAt = 10L; + when(system2.now()).thenReturn(createdAt); + + InternalComponentPropertyDto dto = saveDto(); + + long updatedAt = 20L; + when(system2.now()).thenReturn(updatedAt); + + String newValue = "newValue"; + underTest.insertOrUpdate(dbSession, dto.getComponentUuid(), dto.getKey(), newValue); + + InternalComponentPropertyDto updatedDto = underTest.selectByComponentUuidAndKey(dbSession, dto.getComponentUuid(), dto.getKey()).get(); + + assertThat(updatedDto.getComponentUuid()).isEqualTo(SOME_COMPONENT); + assertThat(updatedDto.getKey()).isEqualTo(SOME_KEY); + assertThat(updatedDto.getValue()).isEqualTo(newValue); + assertThat(updatedDto.getUpdatedAt()).isEqualTo(updatedAt); + assertThat(updatedDto.getCreatedAt()).isEqualTo(createdAt); + } + + @Test + public void replaceValue_sets_to_newValue_if_oldValue_matches_expected() { + long createdAt = 10L; + when(system2.now()).thenReturn(createdAt); + InternalComponentPropertyDto dto = saveDto(); + + long updatedAt = 20L; + when(system2.now()).thenReturn(updatedAt); + + String newValue = "other value"; + underTest.replaceValue(dbSession, SOME_COMPONENT, SOME_KEY, SOME_VALUE, newValue); + + InternalComponentPropertyDto updatedDto = underTest.selectByComponentUuidAndKey(dbSession, dto.getComponentUuid(), dto.getKey()).get(); + + assertThat(updatedDto.getValue()).isEqualTo(newValue); + assertThat(updatedDto.getUpdatedAt()).isEqualTo(updatedAt); + assertThat(updatedDto.getCreatedAt()).isEqualTo(createdAt); + } + @Test + public void replaceValue_does_not_replace_if_oldValue_does_not_match_expected() { + long createdAt = 10L; + when(system2.now()).thenReturn(createdAt); InternalComponentPropertyDto dto = saveDto(); - long updateDate = 20L; - when(system2.now()).thenReturn(updateDate); + long updatedAt = 20L; + when(system2.now()).thenReturn(updatedAt); + + underTest.replaceValue(dbSession, SOME_COMPONENT, SOME_KEY, SOME_VALUE + "foo", "other value"); - dto.setValue("other value"); + InternalComponentPropertyDto updatedDto = underTest.selectByComponentUuidAndKey(dbSession, dto.getComponentUuid(), dto.getKey()).get(); - underTest.insertOrUpdate(dbSession, dto); + assertThat(updatedDto.getValue()).isEqualTo(SOME_VALUE); + assertThat(updatedDto.getUpdatedAt()).isEqualTo(createdAt); + assertThat(updatedDto.getCreatedAt()).isEqualTo(createdAt); + } - assertThatInternalProperty(dto.getUuid()) - .hasUpdatedAt(updateDate) - .hasValue("other value") - .hasCreatedAt(creationDate); + @Test + public void replaceValue_does_not_insert_if_record_does_not_exist() { + underTest.replaceValue(dbSession, SOME_COMPONENT, SOME_KEY, SOME_VALUE, "other value"); + assertThat(underTest.selectByComponentUuidAndKey(dbSession, SOME_COMPONENT, SOME_KEY)).isEmpty(); } @Test @@ -128,96 +167,24 @@ public class InternalComponentPropertiesDaoTest { assertThat(underTest.selectByComponentUuidAndKey(dbSession, SOME_COMPONENT, SOME_KEY)).isNotEmpty(); } - private InternalComponentPropertyDto saveDto() { - InternalComponentPropertyDto dto = new InternalComponentPropertyDto() - .setKey(SOME_KEY) - .setComponentUuid(SOME_COMPONENT) - .setValue(SOME_VALUE); - - underTest.insertOrUpdate(dbSession, dto); - return dto; - } - - private InternalComponentPropertyAssert assertThatInternalProperty(String uuid) { - return new InternalComponentPropertiesDaoTest.InternalComponentPropertyAssert(dbTester, dbSession, uuid); + @Test + public void loadDbKey_loads_dbKeys_for_all_components_with_given_property_and_value() { + OrganizationDto organizationDto = dbTester.organizations().insert(); + ComponentDto portfolio1 = dbTester.components().insertPublicPortfolio(organizationDto); + ComponentDto portfolio2 = dbTester.components().insertPublicPortfolio(organizationDto); + ComponentDto portfolio3 = dbTester.components().insertPublicPortfolio(organizationDto); + ComponentDto portfolio4 = dbTester.components().insertPublicPortfolio(organizationDto); + + underTest.insertOrUpdate(dbSession, portfolio1.uuid(), SOME_KEY, SOME_VALUE); + underTest.insertOrUpdate(dbSession, portfolio2.uuid(), SOME_KEY, "bar"); + underTest.insertOrUpdate(dbSession, portfolio3.uuid(), "foo", SOME_VALUE); + + assertThat(underTest.selectDbKeys(dbSession, SOME_KEY, SOME_VALUE)).containsOnly(portfolio1.getDbKey()); } - private static class InternalComponentPropertyAssert extends AbstractAssert { - - public InternalComponentPropertyAssert(DbTester dbTester, DbSession dbSession, String uuid) { - super(asInternalProperty(dbTester, dbSession, uuid), InternalComponentPropertyAssert.class); - } - - private static InternalComponentPropertyDto asInternalProperty(DbTester dbTester, DbSession dbSession, String uuid) { - Map row = dbTester.selectFirst( - dbSession, - "select" + - " uuid as \"uuid\", component_uuid as \"componentUuid\", kee as \"key\", value as \"value\", updated_at as \"updatedAt\", created_at as \"createdAt\"" + - " from internal_component_props" + - " where uuid='" + uuid+ "'"); - return new InternalComponentPropertyDto() - .setUuid((String) row.get("uuid")) - .setComponentUuid((String) row.get("componentUuid")) - .setKey((String) row.get("key")) - .setValue((String) row.get("value")) - .setUpdatedAt((Long) row.get("updatedAt")) - .setCreatedAt((Long) row.get("createdAt")); - } - - public void doesNotExist() { - isNull(); - } - - public InternalComponentPropertiesDaoTest.InternalComponentPropertyAssert hasKey(String expected) { - isNotNull(); - - if (!Objects.equals(actual.getKey(), expected)) { - failWithMessage("Expected Internal property to have column KEY to be <%s> but was <%s>", true, actual.getKey()); - } - - return this; - } - - public InternalComponentPropertiesDaoTest.InternalComponentPropertyAssert hasComponentUuid(String expected) { - isNotNull(); - - if (!Objects.equals(actual.getComponentUuid(), expected)) { - failWithMessage("Expected Internal property to have column COMPONENT_UUID to be <%s> but was <%s>", true, actual.getComponentUuid()); - } - - return this; - } - - public InternalComponentPropertiesDaoTest.InternalComponentPropertyAssert hasValue(String expected) { - isNotNull(); - - if (!Objects.equals(actual.getValue(), expected)) { - failWithMessage("Expected Internal property to have column VALUE to be <%s> but was <%s>", true, actual.getValue()); - } - - return this; - } - - public InternalComponentPropertiesDaoTest.InternalComponentPropertyAssert hasCreatedAt(long expected) { - isNotNull(); - - if (!Objects.equals(actual.getCreatedAt(), expected)) { - failWithMessage("Expected Internal property to have column CREATED_AT to be <%s> but was <%s>", expected, actual.getCreatedAt()); - } - - return this; - } - - public InternalComponentPropertiesDaoTest.InternalComponentPropertyAssert hasUpdatedAt(long expected) { - isNotNull(); - - if (!Objects.equals(actual.getUpdatedAt(), expected)) { - failWithMessage("Expected Internal property to have column UPDATED_AT to be <%s> but was <%s>", expected, actual.getUpdatedAt()); - } - - return this; - } - + private InternalComponentPropertyDto saveDto() { + underTest.insertOrUpdate(dbSession, SOME_COMPONENT, SOME_KEY, SOME_VALUE); + return underTest.selectByComponentUuidAndKey(dbSession, SOME_COMPONENT, SOME_KEY).get(); } } -- 2.39.5