From: Simon Brandhof Date: Mon, 2 Apr 2012 14:21:25 +0000 (+0200) Subject: Improve the way to add the new primary key RESOURCE_INDEX#ID. X-Git-Tag: 3.0~66 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=3d8fd94d37c2b397e413cd7594d765e0f944f7a9;p=sonarqube.git Improve the way to add the new primary key RESOURCE_INDEX#ID. - do not index files during upgrade, but only projects - new webservice POST /api/server/index_projects to index all the existing projects --- diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java index 193091e0f33..9b9131586a9 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java @@ -32,7 +32,7 @@ import java.util.List; */ public class DatabaseVersion implements BatchComponent, ServerComponent { - public static final int LAST_VERSION = 285; + public static final int LAST_VERSION = 286; public static enum Status { UP_TO_DATE, REQUIRES_UPGRADE, REQUIRES_DOWNGRADE, FRESH_INSTALL diff --git a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerDao.java b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerDao.java index d7e26e8a00e..80553c49eed 100644 --- a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerDao.java +++ b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerDao.java @@ -85,10 +85,10 @@ public class ResourceIndexerDao { private void doIndexProject(int rootProjectId, SqlSession session, final ResourceIndexerMapper mapper) { // non indexed resources ResourceIndexerQuery query = ResourceIndexerQuery.create() - .setNonIndexedOnly(true) - .setQualifiers(NOT_RENAMABLE_QUALIFIERS) - .setScopes(NOT_RENAMABLE_SCOPES) - .setRootProjectId(rootProjectId); + .setNonIndexedOnly(true) + .setQualifiers(NOT_RENAMABLE_QUALIFIERS) + .setScopes(NOT_RENAMABLE_SCOPES) + .setRootProjectId(rootProjectId); session.select("org.sonar.core.resource.ResourceIndexerMapper.selectResources", query, new ResultHandler() { public void handleResult(ResultContext context) { @@ -100,10 +100,10 @@ public class ResourceIndexerDao { // some resources can be renamed, so index must be regenerated // -> delete existing rows and create them again query = ResourceIndexerQuery.create() - .setNonIndexedOnly(false) - .setQualifiers(RENAMABLE_QUALIFIERS) - .setScopes(RENAMABLE_SCOPES) - .setRootProjectId(rootProjectId); + .setNonIndexedOnly(false) + .setQualifiers(RENAMABLE_QUALIFIERS) + .setScopes(RENAMABLE_SCOPES) + .setRootProjectId(rootProjectId); session.select("org.sonar.core.resource.ResourceIndexerMapper.selectResources", query, new ResultHandler() { public void handleResult(ResultContext context) { @@ -120,10 +120,10 @@ public class ResourceIndexerDao { String key = nameToKey(resource.getName()); if (key.length() >= MINIMUM_KEY_SIZE) { ResourceIndexDto dto = new ResourceIndexDto() - .setResourceId(resource.getId()) - .setQualifier(resource.getQualifier()) - .setRootProjectId(resource.getRootId()) - .setNameSize(resource.getName().length()); + .setResourceId(resource.getId()) + .setQualifier(resource.getQualifier()) + .setRootProjectId(resource.getRootId()) + .setNameSize(resource.getName().length()); for (int position = 0; position <= key.length() - MINIMUM_KEY_SIZE; position++) { dto.setPosition(position); @@ -133,35 +133,32 @@ public class ResourceIndexerDao { } } - public boolean indexResource(int id, String name, String qualifier, int rootId) { - return indexResource(id, name, qualifier, rootId, false); + public boolean indexResource(long id) { + boolean indexed = false; + SqlSession session = mybatis.openSession(); + try { + ResourceIndexerMapper mapper = session.getMapper(ResourceIndexerMapper.class); + ResourceDto resource = mapper.selectResourceToIndex(id); + if (resource != null) { + Integer rootId = resource.getRootId(); + if (rootId == null) { + rootId = resource.getId().intValue(); + } + indexed = indexResource(resource.getId().intValue(), resource.getName(), resource.getQualifier(), rootId, session, mapper); + } + return indexed; + } finally { + MyBatis.closeQuietly(session); + } } - public boolean indexResource(int id, String name, String qualifier, int rootId, boolean force) { + public boolean indexResource(int id, String name, String qualifier, int rootId) { boolean indexed = false; - if (force || isIndexableQualifier(qualifier)) { + if (isIndexableQualifier(qualifier)) { SqlSession session = mybatis.openSession(); + ResourceIndexerMapper mapper = session.getMapper(ResourceIndexerMapper.class); try { - String key = nameToKey(name); - if (key.length() >= MINIMUM_KEY_SIZE) { - indexed = true; - ResourceIndexerMapper mapper = session.getMapper(ResourceIndexerMapper.class); - boolean toBeIndexed = sanitizeIndex(id, key, mapper); - if (toBeIndexed) { - ResourceIndexDto dto = new ResourceIndexDto() - .setResourceId(id) - .setQualifier(qualifier) - .setRootProjectId(rootId) - .setNameSize(name.length()); - - for (int position = 0; position <= key.length() - MINIMUM_KEY_SIZE; position++) { - dto.setPosition(position); - dto.setKey(StringUtils.substring(key, position)); - mapper.insert(dto); - } - session.commit(); - } - } + indexed = indexResource(id, name, qualifier, rootId, session, mapper); } finally { MyBatis.closeQuietly(session); } @@ -169,6 +166,30 @@ public class ResourceIndexerDao { return indexed; } + private boolean indexResource(int id, String name, String qualifier, int rootId, SqlSession session, ResourceIndexerMapper mapper) { + boolean indexed = false; + String key = nameToKey(name); + if (key.length() >= MINIMUM_KEY_SIZE) { + indexed = true; + boolean toBeIndexed = sanitizeIndex(id, key, mapper); + if (toBeIndexed) { + ResourceIndexDto dto = new ResourceIndexDto() + .setResourceId(id) + .setQualifier(qualifier) + .setRootProjectId(rootId) + .setNameSize(name.length()); + + for (int position = 0; position <= key.length() - MINIMUM_KEY_SIZE; position++) { + dto.setPosition(position); + dto.setKey(StringUtils.substring(key, position)); + mapper.insert(dto); + } + session.commit(); + } + } + return indexed; + } + /** * Return true if the resource must be indexed, false if the resource is already indexed. diff --git a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java index 53f00b440e5..1d5b7d0558c 100644 --- a/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/resource/ResourceIndexerMapper.java @@ -23,6 +23,8 @@ public interface ResourceIndexerMapper { ResourceIndexDto selectMasterIndexByResourceId(long resourceId); + ResourceDto selectResourceToIndex(long resourceId); + void deleteByResourceId(long resourceId); void insert(ResourceIndexDto dto); diff --git a/sonar-core/src/main/resources/org/sonar/core/persistence/rows-derby.sql b/sonar-core/src/main/resources/org/sonar/core/persistence/rows-derby.sql index edfe15b3c26..621b7bda167 100644 --- a/sonar-core/src/main/resources/org/sonar/core/persistence/rows-derby.sql +++ b/sonar-core/src/main/resources/org/sonar/core/persistence/rows-derby.sql @@ -185,6 +185,7 @@ INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('282'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('283'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('284'); INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('285'); +INSERT INTO SCHEMA_MIGRATIONS(VERSION) VALUES ('286'); INSERT INTO USERS(ID, LOGIN, NAME, EMAIL, CRYPTED_PASSWORD, SALT, CREATED_AT, UPDATED_AT, REMEMBER_TOKEN, REMEMBER_TOKEN_EXPIRES_AT) VALUES (1, 'admin', 'Administrator', '', 'a373a0e667abb2604c1fd571eb4ad47fe8cc0878', '48bc4b0d93179b5103fd3885ea9119498e9d161b', '2011-09-26 22:27:48.0', '2011-09-26 22:27:48.0', null, null); ALTER TABLE USERS ALTER COLUMN ID RESTART WITH 2; diff --git a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper-oracle.xml b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper-oracle.xml index f65532665d2..033e59fd0ec 100644 --- a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper-oracle.xml +++ b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper-oracle.xml @@ -47,6 +47,12 @@ where resource_id=#{id} and position=0 + + delete from resource_index where resource_id=#{id} diff --git a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml index 75e37e4f5c1..faf58386640 100644 --- a/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/resource/ResourceIndexerMapper.xml @@ -47,6 +47,12 @@ where resource_id=#{id} and position=0 + + delete from resource_index where resource_id=#{id} diff --git a/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java b/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java index 134ddfb54de..f7e153ac0fa 100644 --- a/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java +++ b/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java @@ -19,8 +19,6 @@ */ package org.sonar.server.ui; -import com.google.common.base.Predicate; -import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import org.apache.commons.configuration.Configuration; import org.slf4j.LoggerFactory; @@ -426,6 +424,10 @@ public final class JRubyFacade { getContainer().getComponentByType(ResourceIndexerDao.class).indexProjects(); } + public void indexResource(long resourceId) { + getContainer().getComponentByType(ResourceIndexerDao.class).indexResource(resourceId); + } + public void deleteResourceTree(long rootProjectId) { getContainer().getComponentByType(PurgeDao.class).deleteResourceTree(rootProjectId); } diff --git a/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/server_controller.rb b/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/server_controller.rb index 93c207d787f..7ab010d6465 100644 --- a/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/server_controller.rb +++ b/sonar-server/src/main/webapp/WEB-INF/app/controllers/api/server_controller.rb @@ -20,13 +20,12 @@ class Api::ServerController < Api::ApiController skip_before_filter :check_authentication, :except => 'system' - before_filter :admin_required, :only => 'system' # prevent HTTP proxies from caching server status before_filter :set_cache_buster, :only => 'index' # execute database setup - verify :method => :post, :only => [ :setup ] + verify :method => :post, :only => [:setup, :index_projects] skip_before_filter :check_database_version, :setup def key @@ -40,14 +39,15 @@ class Api::ServerController < Api::ApiController def index hash={:id => Java::OrgSonarServerPlatform::Platform.getServer().getId(), :version => Java::OrgSonarServerPlatform::Platform.getServer().getVersion()} complete_with_status(hash) - respond_to do |format| + respond_to do |format| format.json{ render :json => jsonp(hash) } format.xml { render :xml => hash.to_xml(:skip_types => true, :root => 'server') } format.text { render :text => text_not_supported} end end - + def system + access_denied unless has_role?(:admin) @server=Server.new json=[ {:system_info => server_properties_to_json(@server.system_info)}, @@ -56,8 +56,8 @@ class Api::ServerController < Api::ApiController {:sonar_plugins => server_properties_to_json(@server.sonar_plugins)}, {:system_properties => server_properties_to_json(@server.system_properties)}, ] - - respond_to do |format| + + respond_to do |format| format.json{ render :json => jsonp(json) } format.xml { render :xml => xml_not_supported } format.text { render :text => text_not_supported} @@ -86,10 +86,16 @@ class Api::ServerController < Api::ApiController end end end - - + + def index_projects + access_denied unless has_role?(:admin) + logger.info 'Indexing projects' + Java::OrgSonarServerUi::JRubyFacade.getInstance().indexProjects() + render_success('Projects indexed') + end + private - + def server_properties_to_json(properties) hash={} properties.each do |prop| diff --git a/sonar-server/src/main/webapp/WEB-INF/db/migrate/241_index_projects.rb b/sonar-server/src/main/webapp/WEB-INF/db/migrate/241_index_projects.rb deleted file mode 100644 index f9eaf960939..00000000000 --- a/sonar-server/src/main/webapp/WEB-INF/db/migrate/241_index_projects.rb +++ /dev/null @@ -1,32 +0,0 @@ -# -# Sonar, entreprise quality control tool. -# Copyright (C) 2008-2012 SonarSource -# mailto:contact AT sonarsource DOT com -# -# Sonar is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# Sonar is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with Sonar; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 -# - -# -# Note: this migration must be executed after 240_delete_resource_orphans. -# -# Sonar 2.13 -# -class IndexProjects < ActiveRecord::Migration - - def self.up - Java::OrgSonarServerUi::JRubyFacade.getInstance().indexProjects() - end - -end diff --git a/sonar-server/src/main/webapp/WEB-INF/db/migrate/285_add_resource_index_primary_key.rb b/sonar-server/src/main/webapp/WEB-INF/db/migrate/285_add_resource_index_primary_key.rb index 7ccf03ecad5..01231a9d2bc 100644 --- a/sonar-server/src/main/webapp/WEB-INF/db/migrate/285_add_resource_index_primary_key.rb +++ b/sonar-server/src/main/webapp/WEB-INF/db/migrate/285_add_resource_index_primary_key.rb @@ -29,14 +29,15 @@ class AddResourceIndexPrimaryKey < ActiveRecord::Migration def self.up ResourceIndex.reset_column_information + # upgrade from version < 2.13 -> the table did not exist and was created in script 237 with the primary key -> ignore + # upgrade from version 2.13 or 2.14 -> the table existed without primary key -> drop and create again unless ResourceIndex.columns_hash.has_key?('id') # Release 2.13 creates the table without primary key. # Unfortunately it's tricky to add a primary key to an existing table, # particularly on Oracle (note that it's perfectly supported on postgresql). # For this reason the table is dropped and created again. - remove_index 'resource_index', :name => 'resource_index_key' - remove_index 'resource_index', :name => 'resource_index_rid' + remove_indices drop_table 'resource_index' create_table 'resource_index' do |t| @@ -48,10 +49,17 @@ class AddResourceIndexPrimaryKey < ActiveRecord::Migration t.column 'qualifier', :string, :limit => 10, :null => false end - say_with_time 'Indexing projects' do - Java::OrgSonarServerUi::JRubyFacade.getInstance().indexProjects() - end + end end + private + def self.remove_indices + begin + remove_index 'resource_index', :name => 'resource_index_key' + remove_index 'resource_index', :name => 'resource_index_rid' + rescue + #ignore + end + end end diff --git a/sonar-server/src/main/webapp/WEB-INF/db/migrate/286_index_projects.rb b/sonar-server/src/main/webapp/WEB-INF/db/migrate/286_index_projects.rb new file mode 100644 index 00000000000..ad29a9a0de7 --- /dev/null +++ b/sonar-server/src/main/webapp/WEB-INF/db/migrate/286_index_projects.rb @@ -0,0 +1,40 @@ +# +# Sonar, entreprise quality control tool. +# Copyright (C) 2008-2012 SonarSource +# mailto:contact AT sonarsource DOT com +# +# Sonar is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# Sonar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with Sonar; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 +# +class IndexProjects < ActiveRecord::Migration + class ResourceIndex < ActiveRecord::Base + set_table_name 'resource_index' + end + + class Project < ActiveRecord::Base + end + + def self.up + ResourceIndex.reset_column_information + Project.reset_column_information + + projects = Project.find(:all, :select => 'id', :conditions => {:enabled => true, :scope => 'PRJ'}) + + say_with_time "Indexing #{projects.size} projects" do + projects.each do |project| + Java::OrgSonarServerUi::JRubyFacade.getInstance().indexResource(project.id) + end + end + end +end