From eae7223b4cdfbce089c4f30a31692ae8130df600 Mon Sep 17 00:00:00 2001 From: simonbrandhof Date: Tue, 20 Dec 2011 21:32:31 +0100 Subject: SONAR-983 resources are indexed during analysis * Rename the column RESOURCE_INDEX.PROJECT_ID to ROOT_PROJECT_ID * Increase the size of RESOURCE_INDEX.PROJECT_ID * Clean resource index from dbcleaner plugin * Experimental: indexes all the existing resources during upgrade from sonar < 2.13 --- .../persistence/resource/ResourceIndexDto.java | 10 +- .../persistence/resource/ResourceIndexerDao.java | 135 ++++++++++++++------- .../resource/ResourceIndexerMapper.java | 10 ++ .../persistence/resource/ResourceIndexerMapper.xml | 25 +++- .../org/sonar/persistence/schema-derby.ddl | 4 +- .../resource/ResourceIndexerDaoTest.java | 40 +++++- .../shouldDeleteIndexes-result.xml | 8 ++ .../ResourceIndexerDaoTest/shouldDeleteIndexes.xml | 31 +++++ .../shouldIndexAllResources-result.xml | 32 +++++ .../shouldIndexAllResources.xml | 13 ++ .../shouldIndexMultiModulesProject-result.xml | 52 ++++++++ .../shouldIndexMultiModulesProject.xml | 19 +++ .../shouldIndexSingleResource-result.xml | 8 ++ .../shouldIndexSingleResource.xml | 1 + .../shouldReindexProjectAfterRenaming-result.xml | 21 ++++ .../shouldReindexProjectAfterRenaming.xml | 14 +++ .../ResourceIndexerDaoTest/testIndex-result.xml | 8 -- .../resource/ResourceIndexerDaoTest/testIndex.xml | 1 - .../ResourceIndexerDaoTest/testIndexAll-result.xml | 32 ----- .../ResourceIndexerDaoTest/testIndexAll.xml | 13 -- 20 files changed, 365 insertions(+), 112 deletions(-) create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming-result.xml create mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming.xml delete mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex-result.xml delete mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex.xml delete mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll-result.xml delete mode 100644 sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll.xml (limited to 'sonar-core/src') diff --git a/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexDto.java b/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexDto.java index 5272747b0ec..e1622a105ba 100644 --- a/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexDto.java +++ b/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexDto.java @@ -25,7 +25,7 @@ public final class ResourceIndexDto { private int position; private int nameSize; private int resourceId; - private int projectId; + private int rootProjectId; public String getKey() { return key; @@ -54,12 +54,12 @@ public final class ResourceIndexDto { return this; } - public int getProjectId() { - return projectId; + public int getRootProjectId() { + return rootProjectId; } - public ResourceIndexDto setProjectId(int i) { - this.projectId = i; + public ResourceIndexDto setRootProjectId(int i) { + this.rootProjectId = i; return this; } diff --git a/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerDao.java b/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerDao.java index 485539a2e5a..3cc2900d356 100644 --- a/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerDao.java +++ b/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerDao.java @@ -19,14 +19,18 @@ */ package org.sonar.persistence.resource; +import com.google.common.collect.Lists; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.apache.ibatis.session.ExecutorType; import org.apache.ibatis.session.ResultContext; import org.apache.ibatis.session.ResultHandler; import org.apache.ibatis.session.SqlSession; +import org.sonar.persistence.DatabaseUtils; import org.sonar.persistence.MyBatis; +import java.util.List; + public class ResourceIndexerDao { public static final int MINIMUM_KEY_SIZE = 3; @@ -37,68 +41,117 @@ public class ResourceIndexerDao { this.mybatis = mybatis; } - void index(ResourceDto resource, SqlSession session) { - String name = resource.getName(); - if (StringUtils.isBlank(name)) { - return; - } - String normalizedName = normalize(name); - if (normalizedName.length() >= MINIMUM_KEY_SIZE) { - ResourceIndexerMapper mapper = session.getMapper(ResourceIndexerMapper.class); - - Integer rootId; - if (resource.getRootId() != null) { - ResourceDto root = mapper.selectRootId(resource.getRootId()); - if (root != null) { - rootId = (Integer) ObjectUtils.defaultIfNull(root.getRootId(), root.getId()); - } else { - rootId = resource.getRootId(); - } - } else { - rootId = resource.getId(); - } - - ResourceIndexDto dto = new ResourceIndexDto() - .setResourceId(resource.getId()) - .setProjectId(rootId) - .setNameSize(name.length()); - - for (int position = 0; position <= normalizedName.length() - MINIMUM_KEY_SIZE; position++) { - dto.setPosition(position); - dto.setKey(StringUtils.substring(normalizedName, position)); - mapper.insert(dto); - } - - session.commit(); - } - } - - public void index(String resourceName, int resourceId, int projectId) { + public ResourceIndexerDao index(String resourceName, int resourceId, int rootProjectId) { SqlSession sqlSession = mybatis.openSession(); try { - index(new ResourceDto().setId(resourceId).setName(resourceName).setRootId(projectId), sqlSession); + index(new ResourceDto().setId(resourceId).setName(resourceName).setRootId(rootProjectId), sqlSession, true); } finally { sqlSession.close(); } + return this; } - public void index(ResourceIndexerFilter filter) { + public ResourceIndexerDao index(ResourceIndexerFilter filter) { final SqlSession sqlSession = mybatis.openSession(ExecutorType.BATCH); try { sqlSession.select("selectResourcesToIndex", filter, new ResultHandler() { public void handleResult(ResultContext context) { ResourceDto resource = (ResourceDto) context.getResultObject(); - index(resource, sqlSession); + + // The column PROJECTS.ROOT_ID references the module but not the root project in a multi-modules project. + boolean correctRootProjectId = false; + + index(resource, sqlSession, correctRootProjectId); } }); } finally { sqlSession.close(); } + return this; + } + + public ResourceIndexerDao delete(List resourceIds) { + final SqlSession sqlSession = mybatis.openSession(); + try { + ResourceIndexerMapper mapper = sqlSession.getMapper(ResourceIndexerMapper.class); + List> partitionsOfResourceIds = Lists.partition(resourceIds, DatabaseUtils.MAX_IN_ELEMENTS); + for (List partitionOfResourceIds : partitionsOfResourceIds) { + if (!partitionOfResourceIds.isEmpty()) { + mapper.deleteByResourceIds(partitionOfResourceIds); + } + } + sqlSession.commit(); + + } finally { + sqlSession.close(); + } + return this; + } + + void index(ResourceDto resource, SqlSession session, boolean correctProjectRootId) { + String name = resource.getName(); + if (StringUtils.isBlank(name) || resource.getId() == null) { + return; + } + + String key = toKey(name); + if (key.length() >= MINIMUM_KEY_SIZE) { + ResourceIndexerMapper mapper = session.getMapper(ResourceIndexerMapper.class); + boolean toBeIndexed = sanitizeIndex(resource, key, mapper); + if (toBeIndexed) { + + ResourceIndexDto dto = new ResourceIndexDto() + .setResourceId(resource.getId()) + .setRootProjectId(loadRootProjectId(resource, mapper, correctProjectRootId)) + .setNameSize(name.length()); + + for (int position = 0; position <= key.length() - MINIMUM_KEY_SIZE; position++) { + dto.setPosition(position); + dto.setKey(StringUtils.substring(key, position)); + mapper.insert(dto); + } + + session.commit(); + } + } + } + + private Integer loadRootProjectId(ResourceDto resource, ResourceIndexerMapper mapper, boolean correctProjectRootId) { + if (correctProjectRootId) { + return resource.getRootId(); + } + Integer rootId; + if (resource.getRootId() != null) { + ResourceDto root = mapper.selectRootId(resource.getRootId()); + if (root != null) { + rootId = (Integer) ObjectUtils.defaultIfNull(root.getRootId(), root.getId()); + } else { + rootId = resource.getRootId(); + } + } else { + rootId = resource.getId(); + } + return rootId; + } + + /** + * Return true if the resource must be indexed, false if the resource is already indexed. + * If the resource is indexed with a different key, then this index is dropped and the + * resource must be indexed again. + */ + private boolean sanitizeIndex(ResourceDto resource, String key, ResourceIndexerMapper mapper) { + ResourceIndexDto masterIndex = mapper.selectMasterIndexByResourceId(resource.getId()); + if (masterIndex != null && !StringUtils.equals(key, masterIndex.getKey())) { + // resource has been renamed -> drop existing indexes + mapper.deleteByResourceId(resource.getId()); + masterIndex = null; + } + return masterIndex == null; } - static String normalize(String input) { + static String toKey(String input) { return StringUtils.lowerCase(input); } } diff --git a/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerMapper.java b/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerMapper.java index 209126ce466..beec37e4bfd 100644 --- a/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerMapper.java +++ b/sonar-core/src/main/java/org/sonar/persistence/resource/ResourceIndexerMapper.java @@ -19,9 +19,19 @@ */ package org.sonar.persistence.resource; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + public interface ResourceIndexerMapper { ResourceDto selectRootId(int id); + ResourceIndexDto selectMasterIndexByResourceId(int resourceId); + + void deleteByResourceId(int resourceId); + + void deleteByResourceIds(@Param("resourceIds") List resourceIds); + void insert(ResourceIndexDto dto); } diff --git a/sonar-core/src/main/resources/org/sonar/persistence/resource/ResourceIndexerMapper.xml b/sonar-core/src/main/resources/org/sonar/persistence/resource/ResourceIndexerMapper.xml index 5b0c389d3d1..94ac6e57a95 100644 --- a/sonar-core/src/main/resources/org/sonar/persistence/resource/ResourceIndexerMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/persistence/resource/ResourceIndexerMapper.xml @@ -3,7 +3,7 @@ - select id, root_id as "rootId", name from projects @@ -18,16 +18,33 @@ - select id, root_id as "rootId" from projects where id=#{id} + + + + delete from resource_index + where resource_id=#{id} + + + + delete from resource_index + where resource_id in + #{i} + + - insert into resource_index (kee, position, name_size, resource_id, project_id) - values (#{key}, #{position}, #{nameSize}, #{resourceId}, #{projectId}) + insert into resource_index (kee, position, name_size, resource_id, root_project_id) + values (#{key}, #{position}, #{nameSize}, #{resourceId}, #{rootProjectId}) diff --git a/sonar-core/src/main/resources/org/sonar/persistence/schema-derby.ddl b/sonar-core/src/main/resources/org/sonar/persistence/schema-derby.ddl index 02ae63063d2..879e426a883 100644 --- a/sonar-core/src/main/resources/org/sonar/persistence/schema-derby.ddl +++ b/sonar-core/src/main/resources/org/sonar/persistence/schema-derby.ddl @@ -460,11 +460,11 @@ CREATE TABLE "LOADED_TEMPLATES" ( ); CREATE TABLE "RESOURCE_INDEX" ( - "KEE" VARCHAR(100) NOT NULL, + "KEE" VARCHAR(400) NOT NULL, "POSITION" INTEGER NOT NULL, "NAME_SIZE" INTEGER NOT NULL, "RESOURCE_ID" INTEGER NOT NULL, - "PROJECT_ID" INTEGER NOT NULL + "ROOT_PROJECT_ID" INTEGER NOT NULL ); CREATE TABLE "ACTION_PLANS" ( diff --git a/sonar-core/src/test/java/org/sonar/persistence/resource/ResourceIndexerDaoTest.java b/sonar-core/src/test/java/org/sonar/persistence/resource/ResourceIndexerDaoTest.java index 08eeaf64efe..3ee2e28b5a3 100644 --- a/sonar-core/src/test/java/org/sonar/persistence/resource/ResourceIndexerDaoTest.java +++ b/sonar-core/src/test/java/org/sonar/persistence/resource/ResourceIndexerDaoTest.java @@ -23,6 +23,8 @@ import org.junit.Before; import org.junit.Test; import org.sonar.persistence.DaoTestCase; +import java.util.Arrays; + public class ResourceIndexerDaoTest extends DaoTestCase { private static ResourceIndexerDao dao; @@ -33,21 +35,47 @@ public class ResourceIndexerDaoTest extends DaoTestCase { } @Test - public void testIndex() { - setupData("testIndex"); + public void shouldIndexSingleResource() { + setupData("shouldIndexSingleResource"); dao.index("ZipUtils", 10, 8); - checkTables("testIndex", "resource_index"); + checkTables("shouldIndexSingleResource", "resource_index"); } @Test - public void testIndexAll() { - setupData("testIndexAll"); + public void shouldIndexAllResources() { + setupData("shouldIndexAllResources"); dao.index(ResourceIndexerFilter.create()); - checkTables("testIndexAll", "resource_index"); + checkTables("shouldIndexAllResources", "resource_index"); } + @Test + public void shouldIndexMultiModulesProject() { + setupData("shouldIndexMultiModulesProject"); + + dao.index(ResourceIndexerFilter.create()); + + checkTables("shouldIndexMultiModulesProject", "resource_index"); + } + + @Test + public void shouldReindexProjectAfterRenaming() { + setupData("shouldReindexProjectAfterRenaming"); + + dao.index(ResourceIndexerFilter.create()); + + checkTables("shouldReindexProjectAfterRenaming", "resource_index"); + } + + @Test + public void shouldDeleteIndexes() { + setupData("shouldDeleteIndexes"); + + dao.delete(Arrays.asList(3, 4, 5, 6)); + + checkTables("shouldDeleteIndexes", "resource_index"); + } } diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes-result.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes-result.xml new file mode 100644 index 00000000000..b863399fac1 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes-result.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes.xml new file mode 100644 index 00000000000..535d54380cb --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldDeleteIndexes.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources-result.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources-result.xml new file mode 100644 index 00000000000..814c0cfbaca --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources-result.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources.xml new file mode 100644 index 00000000000..f89a5e67f25 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexAllResources.xml @@ -0,0 +1,13 @@ + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject-result.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject-result.xml new file mode 100644 index 00000000000..e009e317898 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject-result.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject.xml new file mode 100644 index 00000000000..4dabdc5beb8 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexMultiModulesProject.xml @@ -0,0 +1,19 @@ + + + + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource-result.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource-result.xml new file mode 100644 index 00000000000..f0b879c3925 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource-result.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource.xml new file mode 100644 index 00000000000..5a4a28b7df3 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldIndexSingleResource.xml @@ -0,0 +1 @@ + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming-result.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming-result.xml new file mode 100644 index 00000000000..23893f22365 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming-result.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming.xml new file mode 100644 index 00000000000..e29238680c8 --- /dev/null +++ b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/shouldReindexProjectAfterRenaming.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex-result.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex-result.xml deleted file mode 100644 index 3c1544c8f9a..00000000000 --- a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex-result.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex.xml deleted file mode 100644 index 5ed00ba028b..00000000000 --- a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndex.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll-result.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll-result.xml deleted file mode 100644 index c1dd1bf9944..00000000000 --- a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll-result.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll.xml b/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll.xml deleted file mode 100644 index f89a5e67f25..00000000000 --- a/sonar-core/src/test/resources/org/sonar/persistence/resource/ResourceIndexerDaoTest/testIndexAll.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - -- cgit v1.2.3