import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
import org.elasticsearch.action.index.IndexRequest;
import org.sonar.api.Startable;
import org.sonar.db.DbClient;
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.EsClient;
+import static java.util.Objects.requireNonNull;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_COMPONENTS;
import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_AUTHORIZATION;
import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT;
/**
* Copy all components of all projects to the elastic search index.
* <p>
- * <b>Warning</b>: This should only be called on an empty index. It does not delete anything.
+ * <b>Warning</b>: This should only be called on an empty index and it should only be called during startup.
*/
public void index() {
- try (DbSession dbSession = dbClient.openSession(false)) {
- BulkIndexer bulk = new BulkIndexer(esClient, INDEX_COMPONENTS);
- bulk.setLarge(true);
- bulk.start();
- dbClient.componentDao()
- .selectAll(dbSession, context -> {
- ComponentDto dto = (ComponentDto) context.getResultObject();
- bulk.add(newIndexRequest(toDocument(dto)));
- });
- bulk.stop();
- }
+ doIndexByProjectUuid(null);
}
/**
* Update the index for one specific project. The current data from the database is used.
*/
public void indexByProjectUuid(String projectUuid) {
+ requireNonNull(projectUuid);
+ deleteComponentsByProjectUuid(projectUuid);
+ doIndexByProjectUuid(projectUuid);
+ }
+
+ /**
+ * @param projectUuid the uuid of the project to analyze, or <code>null</code> if all content should be indexed.<br/>
+ * <b>Warning:</b> only use <code>null</code> during startup.
+ */
+ private void doIndexByProjectUuid(@Nullable String projectUuid) {
+ BulkIndexer bulk = new BulkIndexer(esClient, INDEX_COMPONENTS);
+
+ // setLarge must be enabled only during server startup because it disables replicas
+ bulk.setLarge(projectUuid == null);
+
+ bulk.start();
try (DbSession dbSession = dbClient.openSession(false)) {
- deleteComponentsByProjectUuid(projectUuid);
- index(
- dbClient
- .componentDao()
- .selectByProjectUuid(projectUuid, dbSession)
- .toArray(new ComponentDto[0]));
+ dbClient.componentDao()
+ .selectForIndexing(dbSession, projectUuid, context -> {
+ ComponentDto dto = (ComponentDto) context.getResultObject();
+ bulk.add(newIndexRequest(toDocument(dto)));
+ });
}
+ bulk.stop();
}
- public void deleteByProjectUuid(String uuid) {
- deleteComponentsByProjectUuid(uuid);
- deleteAuthorizationByProjectUuid(uuid);
+ public void deleteByProjectUuid(String projectUuid) {
+ requireNonNull(projectUuid);
+ deleteComponentsByProjectUuid(projectUuid);
+ deleteAuthorizationByProjectUuid(projectUuid);
}
private void deleteComponentsByProjectUuid(String projectUuid) {
- esClient
- .prepareDelete(INDEX_COMPONENTS, TYPE_COMPONENT, projectUuid)
- .setRouting(projectUuid)
- .setRefresh(true)
- .get();
+ BulkIndexer.delete(esClient, INDEX_COMPONENTS, esClient.prepareSearch(INDEX_COMPONENTS)
+ .setQuery(boolQuery()
+ .filter(
+ termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid))));
}
private void deleteAuthorizationByProjectUuid(String projectUuid) {
}
@Test
- public void reindex_project() {
+ public void index_and_update_and_reindex_project() {
// insert
ComponentDto component = ComponentTesting.newProjectDto(organization, "UUID-1").setName("OldName");
assertMatches("NewName", 1);
}
+ @Test
+ public void index_and_update_and_reindex_project_with_files() {
+
+ // insert
+ ComponentDto project = dbTester.components().insertProject();
+ ComponentDto file = dbTester.components().insertComponent(ComponentTesting.newFileDto(project).setName("OldFile"));
+
+ // verify insert
+ index(project);
+ assertMatches("OldFile", 1);
+
+ // modify
+ file.setName("NewFile");
+ update(file);
+
+ // verify modification
+ index(project);
+ assertMatches("OldFile", 0);
+ assertMatches("NewFile", 1);
+ }
+
private void insert(ComponentDto component) {
- dbClient.componentDao().insert(dbSession, component);
- dbSession.commit();
+ dbTester.components().insertComponent(component);
}
private void update(ComponentDto component) {
ComponentUpdateDto updateComponent = ComponentUpdateDto.copyFrom(component);
updateComponent.setBChanged(true);
dbClient.componentDao().update(dbSession, updateComponent);
- dbClient.componentDao().applyBChangesForRootComponentUuid(dbSession, "UUID-1");
+ dbClient.componentDao().applyBChangesForRootComponentUuid(dbSession, component.getRootUuid());
dbSession.commit();
}
.setQuery(termQuery(FIELD_NAME, nameQuery))
.get()
.getHits()
- .getHits().length).isEqualTo(numberOfMatches);
+ .getTotalHits()).isEqualTo(numberOfMatches);
}
private ComponentIndexer createIndexer() {
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.ibatis.session.ResultHandler;
}
/**
- * Selects all enabled components. The result is not returned (since it is usually too big), but handed over to the <code>handler</code>
+ * Selects all components that are relevant for indexing. The result is not returned (since it is usually too big), but handed over to the <code>handler</code>
+ * @param session the database session
+ * @param projectUuid the project uuid, which is selected with all of its children
+ * @param handler the action to be applied to every result
*/
- public void selectAll(DbSession session, ResultHandler handler) {
- mapper(session).selectAll(handler);
+ public void selectForIndexing(DbSession session, @Nullable String projectUuid, ResultHandler handler) {
+ Objects.requireNonNull(handler);
+ mapper(session).selectForIndexing(projectUuid, handler);
}
/**
long countGhostProjects(Map<String, Object> parameters);
- void selectAll(ResultHandler handler);
+ void selectForIndexing(@Param("projectUuid") @Nullable String projectUuid, ResultHandler handler);
void insert(ComponentDto componentDto);
</if>
</sql>
- <select id="selectAll" parameterType="map" resultType="Component">
- select distinct
- <include refid="componentColumns"/>
+ <select id="selectForIndexing" parameterType="map" resultType="Component">
+ select
+ <include refid="componentColumns"/>
from projects p
- where p.enabled=${_true}
+ where
+ p.enabled=${_true}
+ and p.copy_component_uuid is null
+ <if test="projectUuid != null">
+ and p.project_uuid=#{projectUuid}
+ </if>
</select>
<insert id="insert" parameterType="Component" keyColumn="id" useGeneratedKeys="true" keyProperty="id">
package org.sonar.db.component;
import com.google.common.base.Optional;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.ibatis.session.ResultContext;
+import org.apache.ibatis.session.ResultHandler;
+import org.assertj.core.api.ListAssert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
assertThat(components).extracting("id").containsOnly(1L, 2L, 3L, 4L);
}
+ @Test
+ public void selectForIndexing_all() {
+ assertSelectForIndexing(null)
+ .doesNotContain("DIS7")
+ .doesNotContain("COPY8")
+ .containsOnly("U1", "U2", "U3", "U4", "U5", "U6");
+ }
+
+ @Test
+ public void selectForIndexing_project() {
+ assertSelectForIndexing("U1")
+ .doesNotContain("DIS7")
+ .doesNotContain("COPY8")
+ .containsOnly("U1", "U2", "U3", "U4");
+ }
+
+ private ListAssert<String> assertSelectForIndexing(@Nullable String projectUuid) {
+ db.prepareDbUnit(getClass(), "selectForIndexing.xml");
+
+ List<ComponentDto> components = new ArrayList<>();
+ underTest.selectForIndexing(dbSession, projectUuid, new ResultHandler() {
+
+ @Override
+ public void handleResult(ResultContext context) {
+ components.add((ComponentDto) context.getResultObject());
+ }
+ });
+ return assertThat(components).extracting(ComponentDto::uuid);
+ }
+
@Test
public void insert() {
db.prepareDbUnit(getClass(), "empty.xml");
--- /dev/null
+<dataset>
+
+ <organizations uuid="org1"
+ kee="org1_key"
+ name="org1_name"
+ created_at="1000"
+ updated_at="1000"/>
+
+ <!-- root project -->
+ <projects organization_uuid="org1"
+ id="1"
+ scope="PRJ"
+ qualifier="TRK"
+ kee="org.struts:struts"
+ deprecated_kee="org.struts:struts"
+ name="Struts"
+ uuid="U1"
+ uuid_path="uuid_path_of_U1"
+ root_uuid="U1"
+ project_uuid="U1"
+ module_uuid="module_uuid_of_U1"
+ module_uuid_path="module_uuid_path_of_U1"
+ description="the description"
+ long_name="Apache Struts"
+ enabled="[true]"
+ language="java"
+ copy_component_uuid="[null]"
+ developer_uuid="[null]"
+ path="path_of_U1"
+ authorization_updated_at="123456789"/>
+
+ <!-- module -->
+ <projects organization_uuid="org1"
+ id="2"
+ kee="org.struts:struts-core"
+ name="Struts Core"
+ uuid="U2"
+ uuid_path="uuid_path_of_U2"
+ project_uuid="U1"
+ root_uuid="U1"
+ module_uuid="[null]"
+ module_uuid_path="module_uuid_path_of_U2"
+ scope="PRJ"
+ qualifier="BRC"
+ long_name="Struts Core"
+ description="[null]"
+ enabled="[true]"
+ language="[null]"
+ copy_component_uuid="[null]"
+ developer_uuid="[null]"
+ authorization_updated_at="[null]"/>
+
+ <!-- directory -->
+ <projects organization_uuid="org1"
+ long_name="org.struts"
+ id="3"
+ scope="DIR"
+ qualifier="DIR"
+ kee="org.struts:struts-core:src/org/struts"
+ uuid="U3"
+ uuid_path="uuid_path_of_U3"
+ project_uuid="U1"
+ root_uuid="U1"
+ module_uuid="module_uuid_of_U3"
+ module_uuid_path="module_uuid_path_of_U3"
+ name="src/org/struts"
+ description="[null]"
+ enabled="[true]"
+ language="[null]"
+ copy_component_uuid="[null]"
+ developer_uuid="[null]"
+ path="src/org/struts"
+ authorization_updated_at="[null]"/>
+
+ <!-- file -->
+ <projects organization_uuid="org1"
+ long_name="org.struts.RequestContext"
+ id="4"
+ scope="FIL"
+ qualifier="FIL"
+ kee="org.struts:struts-core:src/org/struts/RequestContext.java"
+ uuid="U4"
+ uuid_path="uuid_path_of_U4"
+ project_uuid="U1"
+ root_uuid="U1"
+ module_uuid="module_uuid_of_U4"
+ module_uuid_path="module_uuid_path_of_U4"
+ name="RequestContext.java"
+ description="[null]"
+ enabled="[true]"
+ language="java"
+ copy_component_uuid="[null]"
+ developer_uuid="[null]"
+ path="path_of_U4"
+ authorization_updated_at="[null]"/>
+
+ <!-- root project -->
+ <projects organization_uuid="org1"
+ id="5"
+ scope="PRJ"
+ qualifier="TRK"
+ kee="org.paper:paper"
+ deprecated_kee="org.paper:paper"
+ name="Paper"
+ uuid="U5"
+ uuid_path="uuid_path_of_U5"
+ root_uuid="U5"
+ project_uuid="U5"
+ module_uuid="module_uuid_of_U5"
+ module_uuid_path="module_uuid_path_of_U5"
+ description="the description"
+ long_name="Some Paper"
+ enabled="[true]"
+ language="java"
+ copy_component_uuid="[null]"
+ developer_uuid="[null]"
+ path="path_of_U5"
+ authorization_updated_at="123456789"/>
+
+ <!-- module -->
+ <projects organization_uuid="org1"
+ id="6"
+ kee="org.paper:paper-core"
+ name="Paper Core"
+ uuid="U6"
+ uuid_path="uuid_path_of_U6"
+ project_uuid="U5"
+ root_uuid="U5"
+ module_uuid="[null]"
+ module_uuid_path="module_uuid_path_of_U6"
+ scope="PRJ"
+ qualifier="BRC"
+ long_name="Paper Core"
+ description="[null]"
+ enabled="[true]"
+ language="[null]"
+ copy_component_uuid="[null]"
+ developer_uuid="[null]"
+ authorization_updated_at="[null]"/>
+
+ <!-- Disabled projects -->
+ <projects organization_uuid="org1"
+ id="7"
+ scope="PRJ"
+ qualifier="TRK"
+ kee="org.disabled.project"
+ name="Disabled Project"
+ uuid="DIS7"
+ uuid_path="uuid_path_of_DIS7"
+ project_uuid="project_uuid_of_DIS7"
+ root_uuid="root_uuid_of_DIS7"
+ module_uuid="[null]"
+ module_uuid_path="module_uuid_path_of_DIS7"
+ description="the description"
+ long_name="Disabled project"
+ enabled="[false]"
+ language="[null]"
+ copy_component_uuid="[null]"
+ developer_uuid="[null]"
+ path="[null]"
+ authorization_updated_at="[null]"/>
+
+ <!-- copy component projects -->
+ <projects organization_uuid="org1"
+ id="8"
+ scope="PRJ"
+ qualifier="TRK"
+ kee="org.copy.project"
+ name="Copy Project"
+ uuid="COPY8"
+ uuid_path="uuid_path_of_COPY8"
+ project_uuid="project_uuid_of_COPY8"
+ root_uuid="root_uuid_of_COPY8"
+ module_uuid="[null]"
+ module_uuid_path="module_uuid_path_of_COPY8"
+ description="the description"
+ long_name="Copy project"
+ enabled="[true]"
+ language="[null]"
+ copy_component_uuid="U1"
+ developer_uuid="[null]"
+ path="[null]"
+ authorization_updated_at="[null]"/>
+
+</dataset>
projects / sonarqube.git / commitdiff