diff options
author | Daniel Schwarz <daniel.schwarz@sonarsource.com> | 2017-02-22 10:52:46 +0100 |
---|---|---|
committer | Daniel Schwarz <bartfastiel@users.noreply.github.com> | 2017-02-24 17:21:23 +0100 |
commit | 0050be937f7655a5c0f0583bbefd0137e5bf28e3 (patch) | |
tree | b1b267355c7c62fb9a969eae4fde9523935036dd | |
parent | 5e5e3912f7e5cdff3799c40941d0a382f15c5b9c (diff) | |
download | sonarqube-0050be937f7655a5c0f0583bbefd0137e5bf28e3.tar.gz sonarqube-0050be937f7655a5c0f0583bbefd0137e5bf28e3.zip |
SONAR-8092 index issues when changed in the db, ignore updatedAt
33 files changed, 703 insertions, 392 deletions
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java index 56376179c60..96142a0ebb3 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java @@ -78,6 +78,7 @@ import org.sonar.server.event.NewAlerts; import org.sonar.server.issue.IssueFieldsSetter; import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.issue.notification.ChangesOnMyIssueNotificationDispatcher; import org.sonar.server.issue.notification.DoNotFixNotificationDispatcher; import org.sonar.server.issue.notification.IssueChangesEmailTemplate; @@ -349,6 +350,7 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { // issues IssueIndexer.class, + IssueIteratorFactory.class, PermissionIndexer.class, IssueFieldsSetter.class, // used in Web Services and CE's DebtCalculator FunctionExecutor.class, // used by IssueWorkflow diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java index 1d005bfc1a1..39fef5d400f 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java @@ -88,7 +88,7 @@ public class ComputeEngineContainerImplTest { assertThat(picoContainer.getComponentAdapters()) .hasSize( CONTAINER_ITSELF - + 78 // level 4 + + 79 // level 4 + 4 // content of CeConfigurationModule + 3 // content of CeHttpModule + 5 // content of CeQueueModule diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java b/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java index 9afd83c3135..97ae51697f9 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java @@ -152,7 +152,7 @@ public class DatabaseUtils { /** * Ensure values {@code inputs} are unique (which avoids useless arguments) and sorted before creating the partition. */ - private static <INPUT extends Comparable<INPUT>> Iterable<List<INPUT>> toUniqueAndSortedPartitions(Collection<INPUT> inputs) { + public static <INPUT extends Comparable<INPUT>> Iterable<List<INPUT>> toUniqueAndSortedPartitions(Collection<INPUT> inputs) { return Iterables.partition(toUniqueAndSortedList(inputs), PARTITION_SIZE_FOR_ORACLE); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java index 4555aad7e3e..094bd2b694e 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java @@ -55,7 +55,7 @@ public class IssueDao implements Dao { * if input keys contain multiple occurrences of a key. * <p>Results may be in a different order as input keys (see {@link #selectByOrderedKeys(DbSession, List)}).</p> */ - public List<IssueDto> selectByKeys(final DbSession session, List<String> keys) { + public List<IssueDto> selectByKeys(final DbSession session, Collection<String> keys) { return executeLargeInputs(keys, mapper(session)::selectByKeys); } @@ -63,7 +63,7 @@ public class IssueDao implements Dao { * Gets a list issues by their keys. The result does NOT contain {@code null} values for issues not found, so * the size of result may be less than the number of keys. A single issue is returned * if input keys contain multiple occurrences of a key. - * <p>Contrary to {@link #selectByKeys(DbSession, List)}, results are in the same order as input keys.</p> + * <p>Contrary to {@link #selectByKeys(DbSession, Collection)}, results are in the same order as input keys.</p> */ public List<IssueDto> selectByOrderedKeys(DbSession session, List<String> keys) { List<IssueDto> unordered = selectByKeys(session, keys); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java index e6547312bb9..a4684110065 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java @@ -66,8 +66,6 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator<ProjectMea "LEFT OUTER JOIN snapshots s ON s.component_uuid=p.uuid AND s.islast=? " + "WHERE p.enabled=? AND p.scope=? AND p.qualifier=?"; - private static final String DATE_FILTER = " AND s.created_at>?"; - private static final String PROJECT_FILTER = " AND p.uuid=?"; private static final String SQL_METRICS = "SELECT m.id, m.name FROM metrics m " + @@ -90,12 +88,12 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator<ProjectMea this.projects = projects.iterator(); } - public static ProjectMeasuresIndexerIterator create(DbSession session, long afterDate, @Nullable String projectUuid) { + public static ProjectMeasuresIndexerIterator create(DbSession session, @Nullable String projectUuid) { try { Map<Long, String> metrics = selectMetricKeysByIds(session); - List<Project> projects = selectProjects(session, afterDate, projectUuid); - PreparedStatement measuresStatement = createMeasuresStatement(session, metrics.keySet()); - return new ProjectMeasuresIndexerIterator(measuresStatement, metrics, projects); + List<Project> projects = selectProjects(session, projectUuid); + PreparedStatement projectsStatement = createMeasuresStatement(session, metrics.keySet()); + return new ProjectMeasuresIndexerIterator(projectsStatement, metrics, projects); } catch (SQLException e) { throw new IllegalStateException("Fail to execute request to select all project measures", e); } @@ -121,9 +119,9 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator<ProjectMea return stmt; } - private static List<Project> selectProjects(DbSession session, long afterDate, @Nullable String projectUuid) { + private static List<Project> selectProjects(DbSession session, @Nullable String projectUuid) { List<Project> projects = new ArrayList<>(); - try (PreparedStatement stmt = createProjectsStatement(session, afterDate, projectUuid); + try (PreparedStatement stmt = createProjectsStatement(session, projectUuid); ResultSet rs = stmt.executeQuery()) { while (rs.next()) { String orgUuid = rs.getString(1); @@ -141,23 +139,17 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator<ProjectMea } } - private static PreparedStatement createProjectsStatement(DbSession session, long afterDate, @Nullable String projectUuid) { + private static PreparedStatement createProjectsStatement(DbSession session, @Nullable String projectUuid) { try { String sql = SQL_PROJECTS; - sql += afterDate <= 0L ? "" : DATE_FILTER; sql += projectUuid == null ? "" : PROJECT_FILTER; PreparedStatement stmt = session.getConnection().prepareStatement(sql); stmt.setBoolean(1, true); stmt.setBoolean(2, true); stmt.setString(3, Scopes.PROJECT); stmt.setString(4, Qualifiers.PROJECT); - int index = 5; - if (afterDate > 0L) { - stmt.setLong(index, afterDate); - index++; - } if (projectUuid != null) { - stmt.setString(index, projectUuid); + stmt.setString(5, projectUuid); } return stmt; } catch (SQLException e) { diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java index a496053e635..84f5cc677dc 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java @@ -219,7 +219,7 @@ public class ProjectMeasuresIndexerIteratorTest { dbTester.components().insertProjectAndSnapshot(newProjectDto(organizationDto)); dbTester.components().insertProjectAndSnapshot(newProjectDto(organizationDto)); - Map<String, ProjectMeasures> docsById = createResultSetAndReturnDocsById(0L, project.uuid()); + Map<String, ProjectMeasures> docsById = createResultSetAndReturnDocsById(project.uuid()); assertThat(docsById).hasSize(1); ProjectMeasures doc = docsById.get(project.uuid()); @@ -231,37 +231,20 @@ public class ProjectMeasuresIndexerIteratorTest { } @Test - public void return_only_docs_after_date() throws Exception { - OrganizationDto organizationDto = dbTester.organizations().insert(); - ComponentDto project1 = newProjectDto(organizationDto); - dbClient.componentDao().insert(dbSession, project1); - dbClient.snapshotDao().insert(dbSession, newAnalysis(project1).setCreatedAt(1_000_000L)); - ComponentDto project2 = newProjectDto(organizationDto); - dbClient.componentDao().insert(dbSession, project2); - dbClient.snapshotDao().insert(dbSession, newAnalysis(project2).setCreatedAt(2_000_000L)); - dbSession.commit(); - - Map<String, ProjectMeasures> docsById = createResultSetAndReturnDocsById(1_500_000L, null); - - assertThat(docsById).hasSize(1); - assertThat(docsById.get(project2.uuid())).isNotNull(); - } - - @Test public void return_nothing_on_unknown_project() throws Exception { dbTester.components().insertProjectAndSnapshot(newProjectDto(dbTester.getDefaultOrganization())); - Map<String, ProjectMeasures> docsById = createResultSetAndReturnDocsById(0L, "UNKNOWN"); + Map<String, ProjectMeasures> docsById = createResultSetAndReturnDocsById("UNKNOWN"); assertThat(docsById).isEmpty(); } private Map<String, ProjectMeasures> createResultSetAndReturnDocsById() { - return createResultSetAndReturnDocsById(0L, null); + return createResultSetAndReturnDocsById(null); } - private Map<String, ProjectMeasures> createResultSetAndReturnDocsById(long date, @Nullable String projectUuid) { - ProjectMeasuresIndexerIterator it = ProjectMeasuresIndexerIterator.create(dbTester.getSession(), date, projectUuid); + private Map<String, ProjectMeasures> createResultSetAndReturnDocsById(@Nullable String projectUuid) { + ProjectMeasuresIndexerIterator it = ProjectMeasuresIndexerIterator.create(dbTester.getSession(), projectUuid); Map<String, ProjectMeasures> docsById = Maps.uniqueIndex(it, pm -> pm.getProject().getUuid()); it.close(); return docsById; diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueStorage.java b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueStorage.java index 10a867f9199..90b515d981b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueStorage.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueStorage.java @@ -19,7 +19,12 @@ */ package org.sonar.server.issue; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.sonar.api.issue.Issue; import org.sonar.api.issue.IssueComment; import org.sonar.api.rules.Rule; @@ -35,7 +40,9 @@ import org.sonar.db.MyBatis; import org.sonar.db.issue.IssueChangeDto; import org.sonar.db.issue.IssueChangeMapper; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.Lists.newArrayList; +import static java.util.Collections.emptyList; /** * Save issues into database. It is executed : @@ -83,45 +90,58 @@ public abstract class IssueStorage { // Batch session can not be used for updates. It does not return the number of updated rows, // required for detecting conflicts. long now = system2.now(); - List<DefaultIssue> toBeUpdated = batchInsertAndReturnIssuesToUpdate(session, issues, now); - update(toBeUpdated, now); - doAfterSave(issues); + + Map<Boolean, List<DefaultIssue>> issuesNewOrUpdated = StreamSupport.stream(issues.spliterator(), true).collect(Collectors.groupingBy(DefaultIssue::isNew)); + List<DefaultIssue> issuesToInsert = firstNonNull(issuesNewOrUpdated.get(true), emptyList()); + List<DefaultIssue> issuesToUpdate = firstNonNull(issuesNewOrUpdated.get(false), emptyList()); + + Collection<String> inserted = insert(session, issuesToInsert, now); + Collection<String> updated = update(issuesToUpdate, now); + + Collection<String> issuesInsertedOrUpdated = new ArrayList<>(issuesToInsert.size() + issuesToUpdate.size()); + issuesInsertedOrUpdated.addAll(inserted); + issuesInsertedOrUpdated.addAll(updated); + doAfterSave(issuesInsertedOrUpdated); } - protected void doAfterSave(Iterable<DefaultIssue> issues) { + protected void doAfterSave(Collection<String> issues) { // overridden on server-side to index ES } - private List<DefaultIssue> batchInsertAndReturnIssuesToUpdate(DbSession session, Iterable<DefaultIssue> issues, long now) { - List<DefaultIssue> toBeUpdated = newArrayList(); + /** + * @return the keys of the inserted issues + */ + private Collection<String> insert(DbSession session, Iterable<DefaultIssue> issuesToInsert, long now) { + List<String> inserted = newArrayList(); int count = 0; IssueChangeMapper issueChangeMapper = session.getMapper(IssueChangeMapper.class); - for (DefaultIssue issue : issues) { - if (issue.isNew()) { - doInsert(session, now, issue); - insertChanges(issueChangeMapper, issue); - count++; - if (count > BatchSession.MAX_BATCH_SIZE) { - session.commit(); - count = 0; - } - } else if (issue.isChanged()) { - toBeUpdated.add(issue); + for (DefaultIssue issue : issuesToInsert) { + String key = doInsert(session, now, issue); + inserted.add(key); + insertChanges(issueChangeMapper, issue); + if (count > BatchSession.MAX_BATCH_SIZE) { + session.commit(); } + count++; } session.commit(); - return toBeUpdated; + return inserted; } - protected abstract void doInsert(DbSession batchSession, long now, DefaultIssue issue); + protected abstract String doInsert(DbSession batchSession, long now, DefaultIssue issue); - private void update(List<DefaultIssue> toBeUpdated, long now) { - if (!toBeUpdated.isEmpty()) { + /** + * @return the keys of the updated issues + */ + private Collection<String> update(List<DefaultIssue> issuesToUpdate, long now) { + Collection<String> updated = new ArrayList<>(); + if (!issuesToUpdate.isEmpty()) { DbSession session = dbClient.openSession(false); try { IssueChangeMapper issueChangeMapper = session.getMapper(IssueChangeMapper.class); - for (DefaultIssue issue : toBeUpdated) { - doUpdate(session, now, issue); + for (DefaultIssue issue : issuesToUpdate) { + String key = doUpdate(session, now, issue); + updated.add(key); insertChanges(issueChangeMapper, issue); } session.commit(); @@ -129,9 +149,10 @@ public abstract class IssueStorage { MyBatis.closeQuietly(session); } } + return updated; } - protected abstract void doUpdate(DbSession batchSession, long now, DefaultIssue issue); + protected abstract String doUpdate(DbSession batchSession, long now, DefaultIssue issue); private void insertChanges(IssueChangeMapper mapper, DefaultIssue issue) { for (IssueComment comment : issue.comments()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java b/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java index 216e45d0ac2..ef46fe55acb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java @@ -19,6 +19,7 @@ */ package org.sonar.server.issue; +import java.util.Collection; import org.sonar.api.rules.RuleFinder; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.System2; @@ -43,25 +44,26 @@ public class ServerIssueStorage extends IssueStorage { } @Override - protected void doInsert(DbSession session, long now, DefaultIssue issue) { + protected String doInsert(DbSession session, long now, DefaultIssue issue) { ComponentDto component = component(session, issue); ComponentDto project = project(session, issue); int ruleId = rule(issue).getId(); IssueDto dto = IssueDto.toDtoForServerInsert(issue, component, project, ruleId, now); getDbClient().issueDao().insert(session, dto); + return dto.getKey(); } @Override - protected void doUpdate(DbSession session, long now, DefaultIssue issue) { + protected String doUpdate(DbSession session, long now, DefaultIssue issue) { IssueDto dto = IssueDto.toDtoForUpdate(issue, now); - getDbClient().issueDao().update(session, dto); + return dto.getKey(); } @Override - protected void doAfterSave(Iterable<DefaultIssue> issues) { - indexer.index(issues); + protected void doAfterSave(Collection<String> issueKeys) { + indexer.index(issueKeys); } protected ComponentDto component(DbSession session, DefaultIssue issue) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index c549f046fd4..450937e2ad5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -20,6 +20,7 @@ package org.sonar.server.issue.index; import com.google.common.collect.ImmutableSet; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -28,8 +29,6 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.sonar.api.resources.Qualifiers; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; @@ -50,12 +49,12 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S private static final int MAX_BATCH_SIZE = 1000; private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_ISSUE, project -> Qualifiers.PROJECT.equals(project.getQualifier())); - private final DbClient dbClient; private final EsClient esClient; + private final IssueIteratorFactory issueIteratorFactory; - public IssueIndexer(DbClient dbClient, EsClient esClient) { - this.dbClient = dbClient; + public IssueIndexer(EsClient esClient, IssueIteratorFactory issueIteratorFactory) { this.esClient = esClient; + this.issueIteratorFactory = issueIteratorFactory; } @Override @@ -73,10 +72,6 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S doIndex(createBulkIndexer(true), (String) null); } - public void indexAll() { - doIndex(createBulkIndexer(false), (String) null); - } - @Override public void indexProject(String projectUuid, Cause cause) { switch (cause) { @@ -101,11 +96,19 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S doIndex(createBulkIndexer(false), issues); } + public void index(Collection<String> issueKeys) { + doIndex(createBulkIndexer(false), issueKeys); + } + + private void doIndex(BulkIndexer bulk, Collection<String> issueKeys) { + try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(issueKeys)) { + doIndex(bulk, issues); + } + } + private void doIndex(BulkIndexer bulk, @Nullable String projectUuid) { - try (DbSession dbSession = dbClient.openSession(false)) { - IssueResultSetIterator rowIt = IssueResultSetIterator.create(dbClient, dbSession, projectUuid); - doIndex(bulk, rowIt); - rowIt.close(); + try (IssueIterator issues = issueIteratorFactory.createForProject(projectUuid)) { + doIndex(bulk, issues); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIterator.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIterator.java new file mode 100644 index 00000000000..74ab939763a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIterator.java @@ -0,0 +1,29 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import java.util.Iterator; + +public interface IssueIterator extends Iterator<IssueDoc>, AutoCloseable { + + @Override + void close(); + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorFactory.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorFactory.java new file mode 100644 index 00000000000..93332b4305a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorFactory.java @@ -0,0 +1,45 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import java.util.Collection; +import javax.annotation.Nullable; +import org.sonar.db.DbClient; + +public class IssueIteratorFactory { + + private final DbClient dbClient; + + public IssueIteratorFactory(DbClient dbClient) { + this.dbClient = dbClient; + } + + public IssueIterator createForAll() { + return createForProject((String) null); + } + + public IssueIterator createForProject(@Nullable String projectUuid) { + return new IssueIteratorForSingleChunk(dbClient, projectUuid, null); + } + + public IssueIterator createForIssueKeys(Collection<String> issueKeys) { + return new IssueIteratorForMultipleChunks(dbClient, issueKeys); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorForMultipleChunks.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorForMultipleChunks.java new file mode 100644 index 00000000000..f8cc7d7339a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorForMultipleChunks.java @@ -0,0 +1,66 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; + +import static java.util.Optional.ofNullable; + +public class IssueIteratorForMultipleChunks implements IssueIterator { + + private final DbClient dbClient; + private final Iterator<List<String>> iteratorOverChunks; + private IssueIteratorForSingleChunk currentChunk; + + public IssueIteratorForMultipleChunks(DbClient dbClient, Collection<String> issueKeys) { + this.dbClient = dbClient; + iteratorOverChunks = DatabaseUtils.toUniqueAndSortedPartitions(issueKeys).iterator(); + } + + @Override + public boolean hasNext() { + if (currentChunk != null && currentChunk.hasNext()) { + return true; + } + return iteratorOverChunks.hasNext(); + } + + @Override + public IssueDoc next() { + if (currentChunk == null || !currentChunk.hasNext()) { + currentChunk = nextChunk(); + } + return currentChunk.next(); + } + + private IssueIteratorForSingleChunk nextChunk() { + List<String> nextInput = iteratorOverChunks.next(); + return new IssueIteratorForSingleChunk(dbClient, null, nextInput); + } + + @Override + public void close() { + ofNullable(currentChunk).ifPresent(IssueIterator::close); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java new file mode 100644 index 00000000000..91f45308ecf --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java @@ -0,0 +1,266 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.Date; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import org.apache.commons.lang.StringUtils; +import org.sonar.api.resources.Scopes; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rules.RuleType; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.ResultSetIterator; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.sonar.api.utils.DateUtils.longToDate; +import static org.sonar.db.DatabaseUtils.getLong; + +/** + * Scrolls over table ISSUES and reads documents to populate + * the issues index + */ +class IssueIteratorForSingleChunk implements IssueIterator { + + private static final String[] FIELDS = { + // column 1 + "i.kee", + "root.uuid", + "i.updated_at", + "i.assignee", + "i.gap", + "i.issue_attributes", + "i.line", + "i.message", + "i.resolution", + "i.severity", + + // column 11 + "i.manual_severity", + "i.checksum", + "i.status", + "i.effort", + "i.author_login", + "i.issue_close_date", + "i.issue_creation_date", + "i.issue_update_date", + "r.plugin_name", + "r.plugin_rule_key", + + // column 21 + "r.language", + "p.uuid", + "p.module_uuid_path", + "p.path", + "p.scope", + "i.tags", + "i.issue_type" + }; + + private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from issues i " + + "inner join rules r on r.id=i.rule_id " + + "inner join projects p on p.uuid=i.component_uuid " + + "inner join projects root on root.uuid=i.project_uuid"; + + private static final String PROJECT_FILTER = " AND root.uuid=?"; + private static final String ISSUE_KEY_FILTER_PREFIX = " AND i.kee IN ("; + private static final String ISSUE_KEY_FILTER_SUFFIX = ")"; + + static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + static final Splitter MODULE_PATH_SPLITTER = Splitter.on('.').trimResults().omitEmptyStrings(); + + private final DbSession session; + + @CheckForNull + private final String projectUuid; + + @CheckForNull + private final Collection<String> issueKeys; + + private final PreparedStatement stmt; + private final ResultSetIterator<IssueDoc> iterator; + + IssueIteratorForSingleChunk(DbClient dbClient, @Nullable String projectUuid, @Nullable Collection<String> issueKeys) { + checkArgument(issueKeys == null || issueKeys.size() <= DatabaseUtils.PARTITION_SIZE_FOR_ORACLE, + "Cannot search for more than " + DatabaseUtils.PARTITION_SIZE_FOR_ORACLE + " issue keys at once. Please provide the keys in smaller chunks."); + this.projectUuid = projectUuid; + this.issueKeys = issueKeys; + this.session = dbClient.openSession(false); + + try { + String sql = createSql(); + stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); + iterator = createIterator(); + } catch (Exception e) { + session.close(); + throw new IllegalStateException("Fail to prepare SQL request to select all issues", e); + } + } + + private IssueIteratorInternal createIterator() { + try { + setParameters(stmt); + return new IssueIteratorInternal(stmt); + } catch (SQLException e) { + DatabaseUtils.closeQuietly(stmt); + throw new IllegalStateException("Fail to prepare SQL request to select all issues", e); + } + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public IssueDoc next() { + return iterator.next(); + } + + private String createSql() { + String sql = SQL_ALL; + sql += projectUuid == null ? "" : PROJECT_FILTER; + if (issueKeys != null && !issueKeys.isEmpty()) { + sql += ISSUE_KEY_FILTER_PREFIX; + sql += IntStream.range(0, issueKeys.size()).mapToObj(i -> "?").collect(Collectors.joining(",")); + sql += ISSUE_KEY_FILTER_SUFFIX; + } + return sql; + } + + private void setParameters(PreparedStatement stmt) throws SQLException { + int index = 1; + if (projectUuid != null) { + stmt.setString(index, projectUuid); + index++; + } + if (issueKeys != null) { + for (String key : issueKeys) { + stmt.setString(index, key); + index++; + } + } + } + + @Override + public void close() { + try { + iterator.close(); + } finally { + DatabaseUtils.closeQuietly(stmt); + session.close(); + } + } + + private static final class IssueIteratorInternal extends ResultSetIterator<IssueDoc> { + + public IssueIteratorInternal(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + @Override + protected IssueDoc read(ResultSet rs) throws SQLException { + IssueDoc doc = new IssueDoc(Maps.newHashMapWithExpectedSize(30)); + + String key = rs.getString(1); + String projectUuid = rs.getString(2); + + // all the fields must be present, even if value is null + doc.setKey(key); + doc.setProjectUuid(projectUuid); + doc.setTechnicalUpdateDate(new Date(rs.getLong(3))); + doc.setAssignee(rs.getString(4)); + doc.setGap(DatabaseUtils.getDouble(rs, 5)); + doc.setAttributes(rs.getString(6)); + doc.setLine(DatabaseUtils.getInt(rs, 7)); + doc.setMessage(rs.getString(8)); + doc.setResolution(rs.getString(9)); + doc.setSeverity(rs.getString(10)); + doc.setManualSeverity(rs.getBoolean(11)); + doc.setChecksum(rs.getString(12)); + doc.setStatus(rs.getString(13)); + doc.setEffort(getLong(rs, 14)); + doc.setAuthorLogin(rs.getString(15)); + doc.setFuncCloseDate(longToDate(getLong(rs, 16))); + doc.setFuncCreationDate(longToDate(getLong(rs, 17))); + doc.setFuncUpdateDate(longToDate(getLong(rs, 18))); + String ruleRepo = rs.getString(19); + String ruleKey = rs.getString(20); + doc.setRuleKey(RuleKey.of(ruleRepo, ruleKey).toString()); + doc.setLanguage(rs.getString(21)); + doc.setComponentUuid(rs.getString(22)); + String moduleUuidPath = rs.getString(23); + doc.setModuleUuid(extractModule(moduleUuidPath)); + doc.setModuleUuidPath(moduleUuidPath); + String scope = rs.getString(25); + String filePath = extractFilePath(rs.getString(24), scope); + doc.setFilePath(filePath); + doc.setDirectoryPath(extractDirPath(doc.filePath(), scope)); + String tags = rs.getString(26); + doc.setTags(ImmutableList.copyOf(IssueIteratorForSingleChunk.TAGS_SPLITTER.split(tags == null ? "" : tags))); + doc.setType(RuleType.valueOf(rs.getInt(27))); + return doc; + } + + @CheckForNull + private static String extractDirPath(@Nullable String filePath, String scope) { + if (filePath != null) { + if (Scopes.DIRECTORY.equals(scope)) { + return filePath; + } + int lastSlashIndex = CharMatcher.anyOf("/").lastIndexIn(filePath); + if (lastSlashIndex > 0) { + return filePath.substring(0, lastSlashIndex); + } + return "/"; + } + return null; + } + + @CheckForNull + private static String extractFilePath(@Nullable String filePath, String scope) { + // On modules, the path contains the relative path of the module starting from its parent, and in E/S we're only interested in the + // path + // of files and directories. + // That's why the file path should be null on modules and projects. + if (filePath != null && !Scopes.PROJECT.equals(scope)) { + return filePath; + } + return null; + } + + private static String extractModule(String moduleUuidPath) { + return Iterators.getLast(IssueIteratorForSingleChunk.MODULE_PATH_SPLITTER.split(moduleUuidPath).iterator()); + } + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java deleted file mode 100644 index 8df279117fd..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2017 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import com.google.common.base.CharMatcher; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterators; -import com.google.common.collect.Maps; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Date; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; -import org.apache.commons.lang.StringUtils; -import org.sonar.api.resources.Scopes; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.rules.RuleType; -import org.sonar.db.DatabaseUtils; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.ResultSetIterator; - -import static org.sonar.api.utils.DateUtils.longToDate; -import static org.sonar.db.DatabaseUtils.getLong; - -/** - * Scrolls over table ISSUES and reads documents to populate - * the issues index - */ -class IssueResultSetIterator extends ResultSetIterator<IssueDoc> { - - private static final String[] FIELDS = { - // column 1 - "i.kee", - "root.uuid", - "i.updated_at", - "i.assignee", - "i.gap", - "i.issue_attributes", - "i.line", - "i.message", - "i.resolution", - "i.severity", - - // column 11 - "i.manual_severity", - "i.checksum", - "i.status", - "i.effort", - "i.author_login", - "i.issue_close_date", - "i.issue_creation_date", - "i.issue_update_date", - "r.plugin_name", - "r.plugin_rule_key", - - // column 21 - "r.language", - "p.uuid", - "p.module_uuid_path", - "p.path", - "p.scope", - "i.tags", - "i.issue_type" - }; - - private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from issues i " + - "inner join rules r on r.id=i.rule_id " + - "inner join projects p on p.uuid=i.component_uuid " + - "inner join projects root on root.uuid=i.project_uuid"; - - private static final String PROJECT_FILTER = " AND root.uuid=?"; - - private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); - - private static final Splitter MODULE_PATH_SPLITTER = Splitter.on('.').trimResults().omitEmptyStrings(); - - private IssueResultSetIterator(PreparedStatement stmt) throws SQLException { - super(stmt); - } - - static IssueResultSetIterator create(DbClient dbClient, DbSession session, @Nullable String projectUuid) { - try { - String sql = SQL_ALL; - sql += projectUuid == null ? "" : PROJECT_FILTER; - PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); - if (projectUuid != null) { - stmt.setString(1, projectUuid); - } - return new IssueResultSetIterator(stmt); - } catch (SQLException e) { - throw new IllegalStateException("Fail to prepare SQL request to select all issues", e); - } - } - - @CheckForNull - private static String extractDirPath(@Nullable String filePath, String scope) { - if (filePath != null) { - if (Scopes.DIRECTORY.equals(scope)) { - return filePath; - } - int lastSlashIndex = CharMatcher.anyOf("/").lastIndexIn(filePath); - if (lastSlashIndex > 0) { - return filePath.substring(0, lastSlashIndex); - } - return "/"; - } - return null; - } - - @CheckForNull - private static String extractFilePath(@Nullable String filePath, String scope) { - // On modules, the path contains the relative path of the module starting from its parent, and in E/S we're only interested in the path - // of files and directories. - // That's why the file path should be null on modules and projects. - if (filePath != null && !Scopes.PROJECT.equals(scope)) { - return filePath; - } - return null; - } - - private static String extractModule(String moduleUuidPath) { - return Iterators.getLast(MODULE_PATH_SPLITTER.split(moduleUuidPath).iterator()); - } - - @Override - protected IssueDoc read(ResultSet rs) throws SQLException { - IssueDoc doc = new IssueDoc(Maps.newHashMapWithExpectedSize(30)); - - String key = rs.getString(1); - String projectUuid = rs.getString(2); - - // all the fields must be present, even if value is null - doc.setKey(key); - doc.setProjectUuid(projectUuid); - doc.setTechnicalUpdateDate(new Date(rs.getLong(3))); - doc.setAssignee(rs.getString(4)); - doc.setGap(DatabaseUtils.getDouble(rs, 5)); - doc.setAttributes(rs.getString(6)); - doc.setLine(DatabaseUtils.getInt(rs, 7)); - doc.setMessage(rs.getString(8)); - doc.setResolution(rs.getString(9)); - doc.setSeverity(rs.getString(10)); - doc.setManualSeverity(rs.getBoolean(11)); - doc.setChecksum(rs.getString(12)); - doc.setStatus(rs.getString(13)); - doc.setEffort(getLong(rs, 14)); - doc.setAuthorLogin(rs.getString(15)); - doc.setFuncCloseDate(longToDate(getLong(rs, 16))); - doc.setFuncCreationDate(longToDate(getLong(rs, 17))); - doc.setFuncUpdateDate(longToDate(getLong(rs, 18))); - String ruleRepo = rs.getString(19); - String ruleKey = rs.getString(20); - doc.setRuleKey(RuleKey.of(ruleRepo, ruleKey).toString()); - doc.setLanguage(rs.getString(21)); - doc.setComponentUuid(rs.getString(22)); - String moduleUuidPath = rs.getString(23); - doc.setModuleUuid(extractModule(moduleUuidPath)); - doc.setModuleUuidPath(moduleUuidPath); - String scope = rs.getString(25); - String filePath = extractFilePath(rs.getString(24), scope); - doc.setFilePath(filePath); - doc.setDirectoryPath(extractDirPath(doc.filePath(), scope)); - String tags = rs.getString(26); - doc.setTags(ImmutableList.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags))); - doc.setType(RuleType.valueOf(rs.getInt(27))); - return doc; - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/ws/BulkChangeAction.java b/server/sonar-server/src/main/java/org/sonar/server/issue/ws/BulkChangeAction.java index f9befa1c0e0..336dc2cc72c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/ws/BulkChangeAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/ws/BulkChangeAction.java @@ -197,10 +197,12 @@ public class BulkChangeAction implements IssuesWsAction { return bulkChangeData -> { BulkChangeResult result = new BulkChangeResult(bulkChangeData.issues.size()); IssueChangeContext issueChangeContext = IssueChangeContext.createUser(new Date(system2.now()), userSession.getLogin()); - bulkChangeData.issues.stream() + + List<DefaultIssue> items = bulkChangeData.issues.stream() .filter(bulkChange(issueChangeContext, bulkChangeData, result)) - .peek(issueStorage::save) - .forEach(sendNotification(issueChangeContext, bulkChangeData)); + .collect(Collectors.toList()); + issueStorage.save(items); + items.stream().forEach(sendNotification(issueChangeContext, bulkChangeData)); return result; }; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index 06387b8b92e..a1e2c36f4c1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -75,7 +75,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization case PROJECT_CREATION: // provisioned projects are supported by WS api/components/search_projects case NEW_ANALYSIS: - doIndex(createBulkIndexer(false), 0L, projectUuid); + doIndex(createBulkIndexer(false), projectUuid); break; default: // defensive case @@ -92,26 +92,20 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization .get(); } - private long doIndex(BulkIndexer bulk, long lastUpdatedAt, @Nullable String projectUuid) { + private void doIndex(BulkIndexer bulk, @Nullable String projectUuid) { try (DbSession dbSession = dbClient.openSession(false); - ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, lastUpdatedAt, projectUuid)) { - return doIndex(bulk, rowIt); + ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { + doIndex(bulk, rowIt); } } - private static long doIndex(BulkIndexer bulk, Iterator<ProjectMeasures> docs) { + private static void doIndex(BulkIndexer bulk, Iterator<ProjectMeasures> docs) { bulk.start(); - long maxDate = 0L; while (docs.hasNext()) { ProjectMeasures doc = docs.next(); bulk.add(newIndexRequest(toProjectMeasuresDoc(doc))); - - Long analysisDate = doc.getProject().getAnalysisDate(); - // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance) - maxDate = Math.max(maxDate, analysisDate == null ? 0L : analysisDate); } bulk.stop(); - return maxDate; } private BulkIndexer createBulkIndexer(boolean large) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 52238ccbcfa..bd92fdce789 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -63,6 +63,7 @@ import org.sonar.server.issue.SetTypeAction; import org.sonar.server.issue.TransitionAction; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.issue.notification.ChangesOnMyIssueNotificationDispatcher; import org.sonar.server.issue.notification.DoNotFixNotificationDispatcher; import org.sonar.server.issue.notification.IssueChangesEmailTemplate; @@ -384,6 +385,7 @@ public class PlatformLevel4 extends PlatformLevel { // issues IssueIndexDefinition.class, IssueIndexer.class, + IssueIteratorFactory.class, PermissionIndexer.class, IssueWsModule.class, NewIssuesEmailTemplate.class, diff --git a/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java index 40cd8f377f9..5af45290d51 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/batch/IssuesActionTest.java @@ -43,6 +43,7 @@ import org.sonar.server.issue.index.IssueDoc; import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.permission.index.AuthorizationTypeSupport; import org.sonar.server.permission.index.PermissionIndexerDao; import org.sonar.server.permission.index.PermissionIndexerTester; @@ -76,7 +77,7 @@ public class IssuesActionTest { @Rule public UserSessionRule userSessionRule = UserSessionRule.standalone(); - private IssueIndexer issueIndexer = new IssueIndexer(db.getDbClient(), es.client()); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), new IssueIteratorFactory(db.getDbClient())); private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es, issueIndexer); private ServerFileSystem fs = mock(ServerFileSystem.class); private WsTester tester; diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java new file mode 100644 index 00000000000..1fc4ca64a29 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java @@ -0,0 +1,79 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import com.google.common.collect.ImmutableSet; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.MapSettings; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE; + +public class IndexerStartupTaskTest { + + private System2 system2 = System2.INSTANCE; + + @Rule + public DbTester db = DbTester.create(system2); + + @Rule + public EsTester es = new EsTester(new FakeIndexDefinition()); + + @Test + public void do_not_reindex_existing_nonempty_indexes() throws Exception { + insertDocumentIntoIndex(); + StartupIndexer indexer = createIndexer(); + + emulateStartup(indexer); + + verify(indexer).getIndexTypes(); + verifyNoMoreInteractions(indexer); + } + + @Test + public void reindex_empty_indexes() throws Exception { + StartupIndexer indexer = createIndexer(); + + emulateStartup(indexer); + + verify(indexer).getIndexTypes(); + verify(indexer).indexOnStartup(); + } + + private void insertDocumentIntoIndex() { + es.putDocuments(INDEX_TYPE_FAKE, new FakeDoc()); + } + + private StartupIndexer createIndexer() { + StartupIndexer indexer = mock(StartupIndexer.class); + doReturn(ImmutableSet.of(INDEX_TYPE_FAKE)).when(indexer).getIndexTypes(); + return indexer; + } + + private void emulateStartup(StartupIndexer indexer) { + new IndexerStartupTask(es.client(), new MapSettings(), indexer).execute(); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index 973e995da88..f4b8c5f5f1e 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -92,7 +92,8 @@ public class IssueServiceMediumTest { } private void index() { - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); } @Test @@ -296,7 +297,7 @@ public class IssueServiceMediumTest { private IssueDto saveIssue(IssueDto issue) { tester.get(IssueDao.class).insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + tester.get(IssueIndexer.class).index(asList(issue.getKey())); return issue; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueStorageTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueStorageTest.java index 533a0fd22e7..51201c20090 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueStorageTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueStorageTest.java @@ -259,17 +259,19 @@ public class IssueStorageTest { } @Override - protected void doInsert(DbSession session, long now, DefaultIssue issue) { + protected String doInsert(DbSession session, long now, DefaultIssue issue) { int ruleId = rule(issue).getId(); IssueDto dto = IssueDto.toDtoForComputationInsert(issue, ruleId, now); session.getMapper(IssueMapper.class).insert(dto); + return dto.getKey(); } @Override - protected void doUpdate(DbSession session, long now, DefaultIssue issue) { + protected String doUpdate(DbSession session, long now, DefaultIssue issue) { IssueDto dto = IssueDto.toDtoForUpdate(issue, now); session.getMapper(IssueMapper.class).update(dto); + return dto.getKey(); } } @@ -285,17 +287,19 @@ public class IssueStorageTest { } @Override - protected void doInsert(DbSession session, long now, DefaultIssue issue) { + protected String doInsert(DbSession session, long now, DefaultIssue issue) { int ruleId = rule(issue).getId(); IssueDto dto = IssueDto.toDtoForServerInsert(issue, component, project, ruleId, now); session.getMapper(IssueMapper.class).insert(dto); + return dto.getKey(); } @Override - protected void doUpdate(DbSession session, long now, DefaultIssue issue) { + protected String doUpdate(DbSession session, long now, DefaultIssue issue) { IssueDto dto = IssueDto.toDtoForUpdate(issue, now); session.getMapper(IssueMapper.class).update(dto); + return dto.getKey(); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueUpdaterTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueUpdaterTest.java index 14b7a155e27..db86b2ee91f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueUpdaterTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueUpdaterTest.java @@ -40,6 +40,7 @@ import org.sonar.db.rule.RuleDto; import org.sonar.server.es.EsTester; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.issue.notification.IssueChangeNotification; import org.sonar.server.notification.NotificationManager; import org.sonar.server.rule.DefaultRuleFinder; @@ -75,9 +76,9 @@ public class IssueUpdaterTest { private NotificationManager notificationManager = mock(NotificationManager.class); private ArgumentCaptor<IssueChangeNotification> notificationArgumentCaptor = ArgumentCaptor.forClass(IssueChangeNotification.class); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbClient)); private IssueUpdater underTest = new IssueUpdater(dbClient, - new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, new IssueIndexer(dbClient, esTester.client())), - notificationManager); + new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, issueIndexer), notificationManager); @Test public void update_issue() throws Exception { diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java index 47f7f4d822b..1baaaaaea91 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java @@ -66,7 +66,7 @@ public class IssueIndexDebtTest { private System2 system2 = System2.INSTANCE; private IssueIndex index; - private IssueIndexer issueIndexer = new IssueIndexer(null, tester.client()); + private IssueIndexer issueIndexer = new IssueIndexer(tester.client(), new IssueIteratorFactory(null)); private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(tester, issueIndexer); @Before diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java index fbeba01d5e2..fde781689c5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java @@ -52,7 +52,7 @@ public class IssueIndexerTest { @Rule public DbTester dbTester = DbTester.create(system2); - private IssueIndexer underTest = new IssueIndexer(dbTester.getDbClient(), esTester.client()); + private IssueIndexer underTest = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbTester.getDbClient())); @Test public void index_on_startup() { @@ -187,7 +187,7 @@ public class IssueIndexerTest { issueDoc.setKey("key"); issueDoc.setTechnicalUpdateDate(new Date()); issueDoc.setProjectUuid("non-exitsing-parent"); - new IssueIndexer(dbTester.getDbClient(), esTester.client()) + new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbTester.getDbClient())) .index(Arrays.asList(issueDoc).iterator()); assertThat(esTester.countDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE)).isEqualTo(1L); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java index 55b82771d0f..1d73ff6f25a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java @@ -39,9 +39,7 @@ public class IssueResultSetIteratorTest { @Test public void iterator_over_one_issue() { dbTester.prepareDbUnit(getClass(), "one_issue.xml"); - IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), null); - Map<String, IssueDoc> issuesByKey = issuesByKey(it); - it.close(); + Map<String, IssueDoc> issuesByKey = issuesByKey(); assertThat(issuesByKey).hasSize(1); @@ -72,9 +70,7 @@ public class IssueResultSetIteratorTest { @Test public void iterator_over_issues() { dbTester.prepareDbUnit(getClass(), "shared.xml"); - IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), null); - Map<String, IssueDoc> issuesByKey = issuesByKey(it); - it.close(); + Map<String, IssueDoc> issuesByKey = issuesByKey(); assertThat(issuesByKey).hasSize(4); @@ -134,9 +130,7 @@ public class IssueResultSetIteratorTest { @Test public void iterator_over_issue_from_project() { dbTester.prepareDbUnit(getClass(), "many_projects.xml"); - IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), "THE_PROJECT_1"); - Map<String, IssueDoc> issuesByKey = issuesByKey(it); - it.close(); + Map<String, IssueDoc> issuesByKey = issuesByKey(factory -> factory.createForProject("THE_PROJECT_1")); assertThat(issuesByKey).hasSize(2); } @@ -144,9 +138,7 @@ public class IssueResultSetIteratorTest { @Test public void extract_directory_path() { dbTester.prepareDbUnit(getClass(), "extract_directory_path.xml"); - IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), null); - Map<String, IssueDoc> issuesByKey = issuesByKey(it); - it.close(); + Map<String, IssueDoc> issuesByKey = issuesByKey(); assertThat(issuesByKey).hasSize(4); @@ -166,9 +158,7 @@ public class IssueResultSetIteratorTest { @Test public void extract_file_path() { dbTester.prepareDbUnit(getClass(), "extract_file_path.xml"); - IssueResultSetIterator it = IssueResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), null); - Map<String, IssueDoc> issuesByKey = issuesByKey(it); - it.close(); + Map<String, IssueDoc> issuesByKey = issuesByKey(); assertThat(issuesByKey).hasSize(4); @@ -185,12 +175,18 @@ public class IssueResultSetIteratorTest { assertThat(issuesByKey.get("FGH").filePath()).isNull(); } - private static Map<String, IssueDoc> issuesByKey(IssueResultSetIterator it) { - return Maps.uniqueIndex(it, new Function<IssueDoc, String>() { - @Override - public String apply(@Nonnull IssueDoc issue) { - return issue.key(); - } - }); + private Map<String, IssueDoc> issuesByKey() { + return issuesByKey(IssueIteratorFactory::createForAll); + } + + private Map<String, IssueDoc> issuesByKey(Function<IssueIteratorFactory, IssueIterator> function) { + try (IssueIterator it = function.apply(new IssueIteratorFactory(dbTester.getDbClient()))) { + return Maps.uniqueIndex(it, new Function<IssueDoc, String>() { + @Override + public String apply(@Nonnull IssueDoc issue) { + return issue.key(); + } + }); + } } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java index 93805a1b438..bd6bc2098f5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java @@ -44,6 +44,7 @@ import org.sonar.server.issue.IssueUpdater; import org.sonar.server.issue.ServerIssueStorage; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.notification.NotificationManager; import org.sonar.server.rule.DefaultRuleFinder; import org.sonar.server.tester.UserSessionRule; @@ -85,8 +86,9 @@ public class AddCommentActionTest { private IssueDbTester issueDbTester = new IssueDbTester(dbTester); - private IssueUpdater issueUpdater = new IssueUpdater(dbClient, - new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, new IssueIndexer(dbClient, esTester.client())), mock(NotificationManager.class)); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbClient)); + private ServerIssueStorage serverIssueStorage = new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, issueIndexer); + private IssueUpdater issueUpdater = new IssueUpdater(dbClient, serverIssueStorage, mock(NotificationManager.class)); private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); private WsActionTester tester = new WsActionTester( diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java index f4656414cf6..c4dff3ab240 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java @@ -51,6 +51,7 @@ import org.sonar.server.issue.ServerIssueStorage; import org.sonar.server.issue.TransitionService; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.issue.notification.IssueChangeNotification; import org.sonar.server.issue.workflow.FunctionExecutor; import org.sonar.server.issue.workflow.IssueWorkflow; @@ -105,7 +106,7 @@ public class BulkChangeActionTest { private IssueFieldsSetter issueFieldsSetter = new IssueFieldsSetter(); private IssueWorkflow issueWorkflow = new IssueWorkflow(new FunctionExecutor(issueFieldsSetter), issueFieldsSetter); - private IssueStorage issueStorage = new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, new IssueIndexer(dbClient, es.client())); + private IssueStorage issueStorage = new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, new IssueIndexer(es.client(), new IssueIteratorFactory(dbClient))); private NotificationManager notificationManager = mock(NotificationManager.class); private List<Action> actions = new ArrayList<>(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java index fdaa4169fd4..766beac7337 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java @@ -47,6 +47,7 @@ import org.sonar.server.issue.ServerIssueStorage; import org.sonar.server.issue.TransitionService; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.issue.workflow.FunctionExecutor; import org.sonar.server.issue.workflow.IssueWorkflow; import org.sonar.server.notification.NotificationManager; @@ -96,8 +97,9 @@ public class DoTransitionActionTest { private IssueWorkflow workflow = new IssueWorkflow(new FunctionExecutor(updater), updater); private TransitionService transitionService = new TransitionService(userSession, workflow); private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbClient)); private IssueUpdater issueUpdater = new IssueUpdater(dbClient, - new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, new IssueIndexer(dbClient, esTester.client())), mock(NotificationManager.class)); + new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, issueIndexer), mock(NotificationManager.class)); private WsAction underTest = new DoTransitionAction(dbClient, userSession, new IssueFinder(dbClient, userSession), issueUpdater, transitionService, responseWriter); private WsActionTester tester = new WsActionTester(underTest); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java index 06c9f67f168..39d409e60d1 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java @@ -137,7 +137,7 @@ public class SearchActionComponentsMediumTest { .setIssueUpdateDate(DateUtils.parseDateTime("2017-12-04T00:00:00+0100")); db.issueDao().insert(session, issue2); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH).execute(); result.assertJson(this.getClass(), "issues_on_different_projects.json"); @@ -154,7 +154,7 @@ public class SearchActionComponentsMediumTest { IssueDto issueInRootModule = IssueTesting.newDto(newRule, project, project).setKee("ISSUE_IN_ROOT_MODULE"); db.issueDao().insert(session, issueInModule, issueInRootModule); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); WsActionTester actionTester = new WsActionTester(tester.get(SearchAction.class)); TestResponse response = actionTester.newRequest() @@ -181,7 +181,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue = IssueTesting.newDto(newRule(), file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_PROJECT_UUIDS, project.uuid()) @@ -223,7 +223,7 @@ public class SearchActionComponentsMediumTest { .setIssueUpdateDate(parseDateTime("2015-10-04T00:00:00+0100")); db.issueDao().insert(session, issueAfterLeak, issueBeforeLeak); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, project.uuid()) @@ -251,7 +251,7 @@ public class SearchActionComponentsMediumTest { .setIssueUpdateDate(parseDateTime("2015-10-04T00:00:00+0100")); db.issueDao().insert(session, issueAfterLeak, issueBeforeLeak); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, project.uuid()) @@ -278,7 +278,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue3 = IssueTesting.newDto(rule, file3, project3).setKee("7b1182fd-b650-4037-80bc-82fd47d4eac2"); db.issueDao().insert(session, issue1, issue2, issue3); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_PROJECT_UUIDS, project1.uuid()) @@ -295,7 +295,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue = IssueTesting.newDto(newRule(), file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_FILE_UUIDS, file.uuid()) @@ -329,7 +329,7 @@ public class SearchActionComponentsMediumTest { IssueDto issueOnTest = IssueTesting.newDto(rule, unitTest, project).setKee("2bd4eac2-b650-4037-80bc-7b1182fd47d4"); db.issueDao().insert(session, issueOnFile, issueOnTest); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENTS, file.key()) @@ -354,7 +354,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue2 = IssueTesting.newDto(newRule, file2, project).setKee("2bd4eac2-b650-4037-80bc-7b1182fd47d4"); db.issueDao().insert(session, issue1, issue2); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, project.uuid()) @@ -373,7 +373,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue = IssueTesting.newDto(newRule(), file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, directory.uuid()) @@ -410,8 +410,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue1 = IssueTesting.newDto(rule, file1, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2"); db.issueDao().insert(session, issue1); session.commit(); - - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, directory1.uuid()) @@ -461,7 +460,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue2 = IssueTesting.newDto(newRule, file2, project).setKee("2bd4eac2-b650-4037-80bc-7b1182fd47d4"); db.issueDao().insert(session, issue1, issue2); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, module.uuid()) @@ -480,7 +479,7 @@ public class SearchActionComponentsMediumTest { IssueDto issue = IssueTesting.newDto(newRule(), file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); userSessionRule.logIn("john"); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) @@ -583,7 +582,7 @@ public class SearchActionComponentsMediumTest { db.issueDao().insert(session, issue1, issue2); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_AUTHORS, "leia") @@ -612,7 +611,7 @@ public class SearchActionComponentsMediumTest { db.issueDao().insert(session, issue1, issue2); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, developer.uuid()) @@ -644,7 +643,7 @@ public class SearchActionComponentsMediumTest { db.issueDao().insert(session, issue1, issue2, issueX); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam(IssuesWsParameters.PARAM_COMPONENT_UUIDS, technicalProject.uuid()) @@ -679,7 +678,7 @@ public class SearchActionComponentsMediumTest { private IssueDto insertIssue(IssueDto issue) { db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + indexIssues(); return issue; } @@ -689,6 +688,11 @@ public class SearchActionComponentsMediumTest { return component; } + private void indexIssues() { + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); + } + private void indexView(String viewUuid, List<String> projects) { tester.get(ViewIndexer.class).index(new ViewDoc().setUuid(viewUuid).setProjects(projects)); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java index 6c2d7f2aeea..d1deef45c79 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java @@ -160,7 +160,8 @@ public class SearchActionMediumTest { .setIssueUpdateDate(DateUtils.parseDateTime("2017-12-04T00:00:00+0100")); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH).execute(); result.assertJson(this.getClass(), "response_contains_all_fields_except_additional_fields.json"); @@ -193,7 +194,8 @@ public class SearchActionMediumTest { .setUserLogin("fabrice") .setCreatedAt(DateUtils.parseDateTime("2014-09-10T12:00:00+0000").getTime())); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john"); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) @@ -229,7 +231,8 @@ public class SearchActionMediumTest { .setUserLogin("fabrice") .setCreatedAt(DateUtils.parseDateTime("2014-09-10T19:10:03+0000").getTime())); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john"); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH).setParam(PARAM_HIDE_COMMENTS, "true").execute(); @@ -251,7 +254,8 @@ public class SearchActionMediumTest { .setAssignee("simon"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam("additionalFields", "_all").execute(); @@ -272,7 +276,8 @@ public class SearchActionMediumTest { .setAssignee("simon"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam("additionalFields", "_all").execute(); @@ -297,7 +302,8 @@ public class SearchActionMediumTest { .setIssueUpdateDate(DateUtils.parseDateTime("2017-12-04T00:00:00+0100")); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .execute(); @@ -312,7 +318,8 @@ public class SearchActionMediumTest { IssueDto issue = IssueTesting.newDto(newRule(), file, project); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH).execute(); assertThat(result.outputAsString()).contains("\"componentId\":" + file.getId() + ","); @@ -329,7 +336,8 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH).setParam(PARAM_COMPONENTS, file.getKey()).execute(); result.assertJson(this.getClass(), "apply_paging_with_one_component.json"); @@ -344,7 +352,8 @@ public class SearchActionMediumTest { IssueDto issue = IssueTesting.newDto(newRule(), file, project); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH).setParam(PARAM_ADDITIONAL_FIELDS, "_all").execute(); result.assertJson(this.getClass(), "components_contains_sub_projects.json"); @@ -364,7 +373,8 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john"); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) @@ -388,7 +398,8 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john"); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) @@ -413,7 +424,8 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john"); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) @@ -471,7 +483,8 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue1, issue2, issue3); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john"); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) @@ -503,7 +516,8 @@ public class SearchActionMediumTest { .setKee("82fd47d4-4037-b650-80bc-7b112bd4eac2"); db.issueDao().insert(session, issue1, issue2, issue3); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam("resolved", "false") @@ -545,7 +559,8 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue1, issue2, issue3); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john-bob.polop"); wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) @@ -572,7 +587,8 @@ public class SearchActionMediumTest { .setKee("82fd47d4-b650-4037-80bc-7b112bd4eac3") .setIssueUpdateDate(DateUtils.parseDateTime("2014-11-03T00:00:00+0100"))); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) .setParam("sort", IssueQuery.SORT_BY_UPDATE_DATE) @@ -592,7 +608,8 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.TestRequest request = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH); request.setParam(WebService.Param.PAGE, "2"); @@ -613,7 +630,8 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.TestRequest request = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH); request.setParam(WebService.Param.PAGE, "1"); @@ -634,7 +652,8 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); WsTester.TestRequest request = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH); request.setParam(PARAM_PAGE_INDEX, "2"); @@ -666,7 +685,8 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue); session.commit(); - tester.get(IssueIndexer.class).indexAll(); + IssueIndexer r = tester.get(IssueIndexer.class); + r.indexOnStartup(r.getIndexTypes()); userSessionRule.logIn("john"); WsTester.Result result = wsTester.newGetRequest(CONTROLLER_ISSUES, ACTION_SEARCH) diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java index bc711474523..7fb4967914c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java @@ -45,6 +45,7 @@ import org.sonar.server.issue.IssueUpdater; import org.sonar.server.issue.ServerIssueStorage; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.notification.NotificationManager; import org.sonar.server.rule.DefaultRuleFinder; import org.sonar.server.tester.UserSessionRule; @@ -88,9 +89,10 @@ public class SetSeverityActionTest { private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbClient)); private WsActionTester tester = new WsActionTester(new SetSeverityAction(userSession, dbClient, new IssueFinder(dbClient, userSession), new IssueFieldsSetter(), new IssueUpdater(dbClient, - new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, new IssueIndexer(dbClient, esTester.client())), mock(NotificationManager.class)), + new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, issueIndexer), mock(NotificationManager.class)), responseWriter)); @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java index 24d35f06729..4fc06bab92c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java @@ -45,6 +45,7 @@ import org.sonar.server.issue.IssueUpdater; import org.sonar.server.issue.ServerIssueStorage; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.notification.NotificationManager; import org.sonar.server.rule.DefaultRuleFinder; import org.sonar.server.tester.UserSessionRule; @@ -88,9 +89,10 @@ public class SetTypeActionTest { private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbClient)); private WsActionTester tester = new WsActionTester(new SetTypeAction(userSession, dbClient, new IssueFinder(dbClient, userSession), new IssueFieldsSetter(), new IssueUpdater(dbClient, - new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, new IssueIndexer(dbClient, esTester.client())), mock(NotificationManager.class)), + new ServerIssueStorage(system2, new DefaultRuleFinder(dbClient), dbClient, issueIndexer), mock(NotificationManager.class)), responseWriter)); @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java index 5e3d33a3f2a..1760758594d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java @@ -45,6 +45,7 @@ import org.sonar.server.issue.index.IssueDoc; import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.issue.index.IssueIteratorFactory; import org.sonar.server.permission.index.AuthorizationTypeSupport; import org.sonar.server.permission.index.PermissionIndexer; import org.sonar.server.tester.UserSessionRule; @@ -52,9 +53,6 @@ import org.sonar.server.tester.UserSessionRule; import static com.google.common.collect.Lists.newArrayList; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; public class ViewIndexerTest { @@ -71,19 +69,11 @@ public class ViewIndexerTest { private DbClient dbClient = dbTester.getDbClient(); private DbSession dbSession = dbTester.getSession(); - private IssueIndexer issueIndexer = new IssueIndexer(dbClient, esTester.client()); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbClient)); private PermissionIndexer permissionIndexer = new PermissionIndexer(dbClient, esTester.client(), issueIndexer); private ViewIndexer underTest = new ViewIndexer(dbClient, esTester.client()); @Test - public void index_on_startup() { - ViewIndexer indexer = spy(underTest); - doNothing().when(indexer).indexOnStartup(); - indexer.indexOnStartup(); - verify(indexer).indexOnStartup(); - } - - @Test public void index_nothing() { underTest.indexOnStartup(); assertThat(esTester.countDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW)).isEqualTo(0L); @@ -107,19 +97,6 @@ public class ViewIndexerTest { } @Test - public void index_only_if_empty_do_nothing_when_index_already_exists() throws Exception { - // Some views are not in the db - dbTester.prepareDbUnit(getClass(), "index.xml"); - esTester.putDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, - new ViewDoc().setUuid("ABCD").setProjects(newArrayList("BCDE"))); - - underTest.indexOnStartup(); - - // ... But they shouldn't be indexed - assertThat(esTester.countDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW)).isEqualTo(1L); - } - - @Test public void index_root_view() { dbTester.prepareDbUnit(getClass(), "index.xml"); @@ -149,14 +126,14 @@ public class ViewIndexerTest { @Test public void clear_views_lookup_cache_on_index_view_uuid() { IssueIndex issueIndex = new IssueIndex(esTester.client(), System2.INSTANCE, userSessionRule, new AuthorizationTypeSupport(userSessionRule)); - IssueIndexer issueIndexer = new IssueIndexer(dbClient, esTester.client()); + IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbClient)); String viewUuid = "ABCD"; RuleDto rule = RuleTesting.newXooX1(); dbClient.ruleDao().insert(dbSession, rule); ComponentDto project1 = addProjectWithIssue(rule, dbTester.organizations().insert()); - issueIndexer.indexAll(); + issueIndexer.indexOnStartup(issueIndexer.getIndexTypes()); permissionIndexer.indexProjectsByUuids(dbSession, asList(project1.uuid())); OrganizationDto organizationDto = dbTester.organizations().insert(); @@ -174,7 +151,7 @@ public class ViewIndexerTest { // Add a project to the view and index it again ComponentDto project2 = addProjectWithIssue(rule, organizationDto); - issueIndexer.indexAll(); + issueIndexer.indexOnStartup(issueIndexer.getIndexTypes()); permissionIndexer.indexProjectsByUuids(dbSession, asList(project2.uuid())); ComponentDto techProject2 = ComponentTesting.newProjectCopy("EFGH", project2, view); |