From: Simon Brandhof Date: Thu, 4 Dec 2014 20:56:18 +0000 (+0100) Subject: Refactor issue indexer (project-agnostic approach) X-Git-Tag: 5.0-RC1~62 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=0da5d5aca58df54a695c5dacb9dee1b0be180756;p=sonarqube.git Refactor issue indexer (project-agnostic approach) --- diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/issue/InitialOpenIssuesSensor.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/issue/InitialOpenIssuesSensor.java index a6089aab064..17f22757db8 100644 --- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/issue/InitialOpenIssuesSensor.java +++ b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/issue/InitialOpenIssuesSensor.java @@ -63,7 +63,7 @@ public class InitialOpenIssuesSensor implements Sensor { @Override public void handleResult(ResultContext rc) { IssueDto dto = (IssueDto) rc.getResultObject(); - dto.setSelectedAt(now); + dto.setSelectedAt(now.getTime()); initialOpenIssuesStack.addIssue(dto); } }); diff --git a/server/sonar-data-test/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java b/server/sonar-data-test/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java index 4a48f050968..771155528f5 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java +++ b/server/sonar-data-test/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java @@ -164,8 +164,8 @@ public class IssueIndexBenchmarkTest { issue.setAssignee(users.next()); issue.setAuthorLogin(users.next()); issue.setLine(RandomUtils.nextInt()); - issue.setCreationDate(new Date()); - issue.setUpdateDate(new Date()); + issue.setTechnicalCreationDate(new Date()); + issue.setTechnicalUpdateDate(new Date()); issue.setFuncUpdateDate(new Date()); issue.setFuncCreationDate(new Date()); issue.setFuncCloseDate(null); diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java index b14b61793d6..7bf18722183 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/ComponentCleanerService.java @@ -65,7 +65,7 @@ public class ComponentCleanerService implements ServerComponent { } private void deleteFromIndices(String projectUuid) { - // optimization : index issues is refreshed once at the end + // optimization : index "issues" is refreshed once at the end issueAuthorizationIndexer.deleteProject(projectUuid, false); issueIndexer.deleteProject(projectUuid, true); sourceLineIndexer.deleteByProject(projectUuid); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/IndexProjectIssuesStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/IndexProjectIssuesStep.java index 34936e7febe..f7391b99638 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/IndexProjectIssuesStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/IndexProjectIssuesStep.java @@ -20,49 +20,31 @@ package org.sonar.server.computation; -import com.google.common.collect.ImmutableMap; import org.sonar.core.component.ComponentDto; import org.sonar.core.computation.db.AnalysisReportDto; import org.sonar.core.persistence.DbSession; -import org.sonar.server.db.DbClient; -import org.sonar.server.issue.index.IssueIndex; -import org.sonar.server.issue.index.IssueNormalizer; -import org.sonar.server.search.IndexClient; - -import java.util.Date; +import org.sonar.server.issue.index.IssueAuthorizationIndexer; +import org.sonar.server.issue.index.IssueIndexer; public class IndexProjectIssuesStep implements ComputationStep { - private final DbClient dbClient; - private final IndexClient index; - public IndexProjectIssuesStep(DbClient dbClient, IndexClient index) { - this.dbClient = dbClient; - this.index = index; + private final IssueAuthorizationIndexer authorizationIndexer; + private final IssueIndexer indexer; + + public IndexProjectIssuesStep(IssueAuthorizationIndexer authorizationIndexer, IssueIndexer indexer) { + this.authorizationIndexer = authorizationIndexer; + this.indexer = indexer; } @Override public void execute(DbSession session, AnalysisReportDto report, ComponentDto project) { - indexProjectIssues(session, project); + authorizationIndexer.index(); + indexer.index(); } @Override public String getDescription() { return "Update issues index"; } - - private void indexProjectIssues(DbSession session, ComponentDto project) { - dbClient.issueDao().synchronizeAfter(session, - getLastIndexSynchronizationDate(project), - parameters(project)); - session.commit(); - } - - private ImmutableMap parameters(ComponentDto project) { - return ImmutableMap.of(IssueNormalizer.IssueField.PROJECT.field(), project.uuid()); - } - - private Date getLastIndexSynchronizationDate(ComponentDto project) { - return index.get(IssueIndex.class).getLastSynchronization(parameters(project)); - } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/DatabaseMigrations.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/DatabaseMigrations.java index 1533b430cfb..6bbc494844d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/DatabaseMigrations.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/DatabaseMigrations.java @@ -71,7 +71,8 @@ public interface DatabaseMigrations { PopulateProjectsUuidColumnsMigration.class, ReplaceIssueFiltersProjectKeyByUuid.class, FeedSnapshotSourcesUpdatedAt.class, - FeedFileSources.class + FeedFileSources.class, + FeedIssueLongDates.class ); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedIssueLongDates.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedIssueLongDates.java new file mode 100644 index 00000000000..cc65ac86564 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedIssueLongDates.java @@ -0,0 +1,64 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.db.migrations.v50; + +import org.sonar.api.utils.System2; +import org.sonar.core.persistence.Database; +import org.sonar.server.db.migrations.BaseDataChange; +import org.sonar.server.db.migrations.MassUpdate; +import org.sonar.server.db.migrations.Select; +import org.sonar.server.db.migrations.SqlStatement; + +import java.sql.SQLException; +import java.util.Date; + +public class FeedIssueLongDates extends BaseDataChange { + + private final System2 system; + + public FeedIssueLongDates(Database db, System2 system) { + super(db); + this.system = system; + } + + @Override + public void execute(Context context) throws SQLException { + final long now = system.now(); + + MassUpdate massUpdate = context.prepareMassUpdate(); + massUpdate.select("SELECT i.id, i.created_at, i.updated_at FROM issues i WHERE created_at_ms IS NULL"); + massUpdate.update("UPDATE issues SET created_at_ms=?, updated_at_ms=? WHERE id=?"); + massUpdate.rowPluralName("issues"); + massUpdate.execute(new MassUpdate.Handler() { + @Override + public boolean handle(Select.Row row, SqlStatement update) throws SQLException { + Long id = row.getLong(1); + Date createdAt = row.getDate(2); + Date updatedAt = row.getDate(3); + + update.setLong(1, Math.min(now, createdAt.getTime())); + update.setLong(2, Math.min(now, updatedAt.getTime())); + update.setLong(3, id); + return true; + } + }); + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index 9dc4caab60c..b61e147b69e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -136,18 +136,24 @@ public class NewIndex { } /** - * Create a inner-field named "sort" with analyzer "sortable" + * Create an inner-field named "sort" with analyzer "sortable" */ public StringFieldBuilder enableSorting() { this.sortable = true; return this; } + /** + * Create an inner-field named "words" with analyzer "words" + */ public StringFieldBuilder enableWordSearch() { this.wordSearch = true; return this; } + /** + * Create a inner-field named "grams" with analyzer "grams" + */ public StringFieldBuilder enableGramSearch() { this.gramSearch = true; return this; diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueBulkChangeService.java b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueBulkChangeService.java index 1146566a937..7383549eabb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueBulkChangeService.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueBulkChangeService.java @@ -130,7 +130,7 @@ public class IssueBulkChangeService { DbSession session = dbClient.openSession(false); try { - List issueDtos = dbClient.issueDao().getByKeys(session, authorizedIssueKeys); + List issueDtos = dbClient.issueDao().selectByKeys(session, authorizedIssueKeys); return newArrayList(Iterables.transform(issueDtos, new Function() { @Override public Issue apply(@Nullable IssueDto input) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java index efa6867f785..5d482b4632d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java @@ -312,7 +312,7 @@ public class IssueService implements ServerComponent { IssueDto getByKeyForUpdate(DbSession session, String key) { // Load from index to check permission : if the user has no permission to see the issue an exception will be generated Issue authorizedIssueIndex = getByKey(key); - return dbClient.issueDao().getByKey(session, authorizedIssueIndex.key()); + return dbClient.issueDao().selectByKey(session, authorizedIssueIndex.key()); } void saveIssue(DbSession session, DefaultIssue issue, IssueChangeContext context, @Nullable String comment) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java b/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java index f10ef8d5c73..4eb8f3164ee 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/ServerIssueStorage.java @@ -28,8 +28,7 @@ import org.sonar.core.issue.db.IssueStorage; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.MyBatis; import org.sonar.server.db.DbClient; - -import java.util.Date; +import org.sonar.server.issue.index.IssueIndexer; /** * @since 3.6 @@ -37,14 +36,16 @@ import java.util.Date; public class ServerIssueStorage extends IssueStorage implements ServerComponent { private final DbClient dbClient; + private final IssueIndexer indexer; - public ServerIssueStorage(MyBatis mybatis, RuleFinder ruleFinder, DbClient dbClient) { + public ServerIssueStorage(MyBatis mybatis, RuleFinder ruleFinder, DbClient dbClient, IssueIndexer indexer) { super(mybatis, ruleFinder); this.dbClient = dbClient; + this.indexer = indexer; } @Override - protected void doInsert(DbSession session, Date now, DefaultIssue issue) { + protected void doInsert(DbSession session, long now, DefaultIssue issue) { ComponentDto component = component(session, issue); ComponentDto project = project(session, issue); int ruleId = ruleId(issue); @@ -54,12 +55,17 @@ public class ServerIssueStorage extends IssueStorage implements ServerComponent } @Override - protected void doUpdate(DbSession session, Date now, DefaultIssue issue) { + protected void doUpdate(DbSession session, long now, DefaultIssue issue) { IssueDto dto = IssueDto.toDtoForUpdate(issue, project(session, issue).getId(), now); dbClient.issueDao().update(session, dto); } + @Override + protected void doAfterSave() { + indexer.index(); + } + protected ComponentDto component(DbSession session, DefaultIssue issue) { return dbClient.componentDao().getByKey(session, issue.componentKey()); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java b/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java index 9ac3f2022ca..0b2aba8a5a8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java @@ -19,72 +19,58 @@ */ package org.sonar.server.issue.db; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.sonar.api.utils.System2; import org.sonar.core.issue.db.IssueDto; import org.sonar.core.issue.db.IssueMapper; import org.sonar.core.persistence.DaoComponent; import org.sonar.core.persistence.DbSession; -import org.sonar.server.db.BaseDao; -import org.sonar.server.issue.index.IssueNormalizer; -import org.sonar.server.search.IndexDefinition; +import org.sonar.core.persistence.MyBatis; +import org.sonar.server.exceptions.NotFoundException; -import javax.annotation.Nullable; +import javax.annotation.CheckForNull; import java.util.Collection; -import java.util.Date; import java.util.List; -import java.util.Map; -public class IssueDao extends BaseDao implements DaoComponent { +public class IssueDao extends org.sonar.core.issue.db.IssueDao implements DaoComponent { - public IssueDao() { - this(System2.INSTANCE); + public IssueDao(MyBatis mybatis) { + super(mybatis); } - @VisibleForTesting - public IssueDao(System2 system) { - super(IndexDefinition.ISSUES, IssueMapper.class, system); - } - - @Override - protected IssueDto doGetNullableByKey(DbSession session, String key) { + @CheckForNull + public IssueDto selectNullableByKey(DbSession session, String key) { return mapper(session).selectByKey(key); } - @Override - protected List doGetByKeys(DbSession session, Collection keys) { - return mapper(session).selectByKeys(keys); + public IssueDto selectByKey(DbSession session, String key) { + IssueDto issue = selectNullableByKey(session, key); + if (issue == null) { + throw new NotFoundException(String.format("Key '%s' not found", key)); + } + return issue; } public List findByActionPlan(DbSession session, String actionPlan) { return mapper(session).selectByActionPlan(actionPlan); } - @Override - protected IssueDto doUpdate(DbSession session, IssueDto issue) { - mapper(session).update(issue); - return issue; + public List selectByKeys(DbSession session, Collection keys) { + return mapper(session).selectByKeys(keys); } - @Override - protected IssueDto doInsert(DbSession session, IssueDto issue) { - Preconditions.checkNotNull(issue.getKey(), "Cannot insert Issue with empty key!"); - Preconditions.checkNotNull(issue.getComponentId(), "Cannot insert Issue with no Component!"); - mapper(session).insert(issue); - return issue; + public void insert(DbSession session, IssueDto dto) { + mapper(session).insert(dto); } - @Override - protected String getSynchronizationStatementName() { - return "selectAfterDate"; + public void insert(DbSession session, IssueDto dto, IssueDto... others) { + IssueMapper mapper = mapper(session); + mapper.insert(dto); + for (IssueDto other : others) { + mapper.insert(other); + } } - @Override - protected Map getSynchronizationParams(@Nullable Date date, Map params) { - Map finalParams = super.getSynchronizationParams(date, params); - finalParams.put(IssueNormalizer.IssueField.PROJECT.field(), params.get(IssueNormalizer.IssueField.PROJECT.field())); - return finalParams; + public void update(DbSession session, IssueDto dto) { + mapper(session).update(dto); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/FakeIssueDto.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/FakeIssueDto.java new file mode 100644 index 00000000000..0db9d5d5b24 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/FakeIssueDto.java @@ -0,0 +1,29 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import org.sonar.core.persistence.Dto; + +public class FakeIssueDto extends Dto { + @Override + public String getKey() { + throw new UnsupportedOperationException(); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java index 5d8a3dbd500..7db14c4895f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java @@ -52,11 +52,11 @@ public class IssueDoc extends BaseDoc implements Issue { @Override public String componentUuid() { - return getField(IssueNormalizer.IssueField.COMPONENT.field()); + return getField(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID); } public String moduleUuid() { - return getField(IssueNormalizer.IssueField.MODULE.field()); + return getField(IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID); } @Override @@ -66,79 +66,85 @@ public class IssueDoc extends BaseDoc implements Issue { @Override public String projectUuid() { - return getField(IssueNormalizer.IssueField.PROJECT.field()); + return getField(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID); } @Override public RuleKey ruleKey() { - return RuleKey.parse((String) getField(IssueNormalizer.IssueField.RULE_KEY.field())); + return RuleKey.parse((String) getField(IssueIndexDefinition.FIELD_ISSUE_RULE_KEY)); } @Override public String language() { - return getField(IssueNormalizer.IssueField.LANGUAGE.field()); + return getField(IssueIndexDefinition.FIELD_ISSUE_LANGUAGE); } @Override public String severity() { - return getField(IssueNormalizer.IssueField.SEVERITY.field()); + return getField(IssueIndexDefinition.FIELD_ISSUE_SEVERITY); } @Override @CheckForNull public String message() { - return getNullableField(IssueNormalizer.IssueField.MESSAGE.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_MESSAGE); } @Override @CheckForNull public Integer line() { - return getNullableField(IssueNormalizer.IssueField.LINE.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_LINE); } @Override @CheckForNull public Double effortToFix() { - return getNullableField(IssueNormalizer.IssueField.EFFORT.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_EFFORT); } @Override public String status() { - return getField(IssueNormalizer.IssueField.STATUS.field()); + return getField(IssueIndexDefinition.FIELD_ISSUE_STATUS); } @Override @CheckForNull public String resolution() { - return getNullableField(IssueNormalizer.IssueField.RESOLUTION.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION); } @Override @CheckForNull public String reporter() { - return getNullableField(IssueNormalizer.IssueField.REPORTER.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_REPORTER); } @Override @CheckForNull public String assignee() { - return getNullableField(IssueNormalizer.IssueField.ASSIGNEE.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE); } + /** + * Functional date + */ @Override public Date creationDate() { - return getFieldAsDate(IssueNormalizer.IssueField.ISSUE_CREATED_AT.field()); + return getFieldAsDate(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT); } + /** + * Functional date + */ @Override public Date updateDate() { - return getFieldAsDate(IssueNormalizer.IssueField.ISSUE_UPDATED_AT.field()); + return getFieldAsDate(IssueIndexDefinition.FIELD_ISSUE_FUNC_UPDATED_AT); } @Override @CheckForNull public Date closeDate() { - return getNullableFieldAsDate(IssueNormalizer.IssueField.ISSUE_CLOSE_DATE.field()); + return getNullableFieldAsDate(IssueIndexDefinition.FIELD_ISSUE_FUNC_CLOSED_AT); } @Override @@ -149,7 +155,7 @@ public class IssueDoc extends BaseDoc implements Issue { @Override public Map attributes() { - String data = getNullableField(IssueNormalizer.IssueField.ATTRIBUTES.field()); + String data = getNullableField(IssueIndexDefinition.FIELD_ISSUE_ATTRIBUTES); if (data == null) { return Collections.emptyMap(); } else { @@ -160,13 +166,13 @@ public class IssueDoc extends BaseDoc implements Issue { @Override @CheckForNull public String authorLogin() { - return getNullableField(IssueNormalizer.IssueField.AUTHOR_LOGIN.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN); } @Override @CheckForNull public String actionPlanKey() { - return getNullableField(IssueNormalizer.IssueField.ACTION_PLAN.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_ACTION_PLAN); } @Override @@ -182,13 +188,13 @@ public class IssueDoc extends BaseDoc implements Issue { @Override @CheckForNull public Duration debt() { - Number debt = getNullableField(IssueNormalizer.IssueField.DEBT.field()); + Number debt = getNullableField(IssueIndexDefinition.FIELD_ISSUE_DEBT); return (debt != null) ? Duration.create(debt.longValue()) : null; } @CheckForNull public String filePath() { - return getNullableField(IssueNormalizer.IssueField.FILE_PATH.field()); + return getNullableField(IssueIndexDefinition.FIELD_ISSUE_FILE_PATH); } public void setKey(@Nullable String s) { @@ -248,20 +254,20 @@ public class IssueDoc extends BaseDoc implements Issue { setField(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE, s); } - public void setCreationDate(@Nullable Date d) { - setField(IssueIndexDefinition.FIELD_ISSUE_CREATED_AT, d); + public void setTechnicalCreationDate(@Nullable Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_CREATED_AT, d); } - public void setUpdateDate(@Nullable Date d) { - setField(IssueIndexDefinition.FIELD_ISSUE_UPDATED_AT, d); + public void setFuncUpdateDate(@Nullable Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_UPDATED_AT, d); } public void setFuncCreationDate(@Nullable Date d) { setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, d); } - public void setFuncUpdateDate(@Nullable Date d) { - setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_UPDATED_AT, d); + public void setTechnicalUpdateDate(@Nullable Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT, d); } public void setFuncCloseDate(@Nullable Date d) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 9dbb9398049..aac89c4cc5f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -45,7 +45,6 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.issue.Issue; import org.sonar.api.rule.Severity; -import org.sonar.core.issue.db.IssueDto; import org.sonar.server.issue.IssueQuery; import org.sonar.server.issue.filter.IssueFilterParameters; import org.sonar.server.search.BaseIndex; @@ -59,7 +58,6 @@ import org.sonar.server.search.StickyFacetBuilder; import org.sonar.server.user.UserSession; import javax.annotation.Nullable; - import java.util.Collection; import java.util.Date; import java.util.List; @@ -68,7 +66,7 @@ import java.util.Set; import static com.google.common.collect.Lists.newArrayList; -public class IssueIndex extends BaseIndex { +public class IssueIndex extends BaseIndex { private static final String FACET_SUFFIX_MISSING = "_missing"; @@ -76,8 +74,8 @@ public class IssueIndex extends BaseIndex { private ListMultimap sortColumns = ArrayListMultimap.create(); - public IssueIndex(IssueNormalizer normalizer, SearchClient client) { - super(IndexDefinition.ISSUES, normalizer, client); + public IssueIndex(SearchClient client) { + super(IndexDefinition.ISSUES, null, client); sortColumns.put(IssueQuery.SORT_BY_ASSIGNEE, IssueNormalizer.IssueField.ASSIGNEE); sortColumns.put(IssueQuery.SORT_BY_STATUS, IssueNormalizer.IssueField.STATUS); @@ -126,15 +124,6 @@ public class IssueIndex extends BaseIndex { return null; } - @Override - protected FilterBuilder getLastSynchronizationBuilder(Map params) { - String projectUuid = params.get(IssueNormalizer.IssueField.PROJECT.field()); - if (projectUuid != null) { - return FilterBuilders.boolFilter().must(FilterBuilders.termsFilter(IssueNormalizer.IssueField.PROJECT.field(), projectUuid)); - } - return super.getLastSynchronizationBuilder(params); - } - public List listAssignees(IssueQuery query) { QueryContext queryContext = new QueryContext().setPage(1, 0); diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index e620fdab286..ca7d0d9ed96 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -25,7 +25,6 @@ import org.sonar.api.config.Settings; import org.sonar.process.ProcessConstants; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; -import org.sonar.server.search.BaseNormalizer; /** * Definition of ES index "issues", including settings and fields. @@ -40,19 +39,31 @@ public class IssueIndexDefinition implements IndexDefinition { public static final String FIELD_AUTHORIZATION_PROJECT_UUID = "project"; public static final String FIELD_AUTHORIZATION_GROUPS = "groups"; public static final String FIELD_AUTHORIZATION_USERS = "users"; - public static final String FIELD_AUTHORIZATION_UPDATED_AT = BaseNormalizer.UPDATED_AT_FIELD; + public static final String FIELD_AUTHORIZATION_UPDATED_AT = "updatedAt"; public static final String FIELD_ISSUE_ACTION_PLAN = "actionPlan"; public static final String FIELD_ISSUE_ASSIGNEE = "assignee"; public static final String FIELD_ISSUE_ATTRIBUTES = "attributes"; public static final String FIELD_ISSUE_AUTHOR_LOGIN = "authorLogin"; public static final String FIELD_ISSUE_COMPONENT_UUID = "component"; - public static final String FIELD_ISSUE_CREATED_AT = "createdAt"; + /** + * Technical date + */ + public static final String FIELD_ISSUE_TECHNICAL_CREATED_AT = "createdAt"; public static final String FIELD_ISSUE_DEBT = "debt"; public static final String FIELD_ISSUE_EFFORT = "effort"; public static final String FIELD_ISSUE_FILE_PATH = "filePath"; + /** + * Functional date + */ public static final String FIELD_ISSUE_FUNC_CREATED_AT = "issueCreatedAt"; + /** + * Functional date + */ public static final String FIELD_ISSUE_FUNC_UPDATED_AT = "issueUpdatedAt"; + /** + * Functional date + */ public static final String FIELD_ISSUE_FUNC_CLOSED_AT = "issueClosedAt"; public static final String FIELD_ISSUE_KEY = "key"; public static final String FIELD_ISSUE_LANGUAGE = "language"; @@ -67,7 +78,10 @@ public class IssueIndexDefinition implements IndexDefinition { public static final String FIELD_ISSUE_SEVERITY = "severity"; public static final String FIELD_ISSUE_SEVERITY_VALUE = "severityValue"; public static final String FIELD_ISSUE_STATUS = "status"; - public static final String FIELD_ISSUE_UPDATED_AT = "updatedAt"; + /** + * Technical date + */ + public static final String FIELD_ISSUE_TECHNICAL_UPDATED_AT = "updatedAt"; private final Settings settings; @@ -131,7 +145,7 @@ public class IssueIndexDefinition implements IndexDefinition { // TODO do we really sort by status ? If yes, then we should sort by "int value", but not by string key issueMapping.stringFieldBuilder(FIELD_ISSUE_STATUS).enableSorting().build(); // TODO is createdAt required ? - issueMapping.createDateTimeField(FIELD_ISSUE_CREATED_AT); - issueMapping.createDateTimeField(FIELD_ISSUE_UPDATED_AT); + issueMapping.createDateTimeField(FIELD_ISSUE_TECHNICAL_CREATED_AT); + issueMapping.createDateTimeField(FIELD_ISSUE_TECHNICAL_UPDATED_AT); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 493635b5531..e43df41dd98 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -44,8 +44,21 @@ public class IssueIndexer extends BaseIndexer { @Override protected long doIndex(long lastUpdatedAt) { - final BulkIndexer bulk = createBulkIndexer(lastUpdatedAt == 0L); + return doIndex(createBulkIndexer(false), lastUpdatedAt); + } + + public void indexAll() { + doIndex(createBulkIndexer(true), 0L); + } + /** + * For benchmarks + */ + public void index(Iterator issues) { + doIndex(createBulkIndexer(false), issues); + } + + private long doIndex(BulkIndexer bulk, long lastUpdatedAt) { DbSession dbSession = dbClient.openSession(false); Connection dbConnection = dbSession.getConnection(); long maxDate; @@ -61,12 +74,7 @@ public class IssueIndexer extends BaseIndexer { } } - public void index(Iterator issues) { - final BulkIndexer bulk = createBulkIndexer(false); - doIndex(bulk, issues); - } - - long doIndex(BulkIndexer bulk, Iterator issues) { + private long doIndex(BulkIndexer bulk, Iterator issues) { bulk.start(); long maxDate = 0L; while (issues.hasNext()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java index 490e9025b4b..1ff008e91be 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java @@ -19,29 +19,26 @@ */ package org.sonar.server.issue.index; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import org.elasticsearch.action.update.UpdateRequest; -import org.sonar.api.rule.Severity; -import org.sonar.core.issue.db.IssueDto; +import org.sonar.core.persistence.Dto; import org.sonar.server.db.DbClient; import org.sonar.server.search.BaseNormalizer; import org.sonar.server.search.IndexField; import org.sonar.server.search.Indexable; import java.util.List; -import java.util.Map; -import java.util.Set; -import static com.google.common.collect.Maps.newHashMap; - -public class IssueNormalizer extends BaseNormalizer { +public class IssueNormalizer extends BaseNormalizer { public IssueNormalizer(DbClient db) { super(db); } + @Override + public List normalize(Dto dto) { + throw new UnsupportedOperationException(); + } + public static final class IssueField extends Indexable { public static final IndexField KEY = addSortable(IndexField.Type.STRING, "key"); @@ -72,60 +69,6 @@ public class IssueNormalizer extends BaseNormalizer { public static final IndexField LANGUAGE = add(IndexField.Type.STRING, "language"); public static final IndexField RULE_KEY = add(IndexField.Type.STRING, "ruleKey"); public static final IndexField FILE_PATH = addSortable(IndexField.Type.STRING, "filePath"); - - public static final Set ALL_FIELDS = ImmutableSet.of(KEY, CREATED_AT, UPDATED_AT, PROJECT, COMPONENT, - MODULE, MODULE_PATH, ACTION_PLAN, ASSIGNEE, ATTRIBUTES, AUTHOR_LOGIN, DEBT, EFFORT, ISSUE_CREATED_AT, - ISSUE_UPDATED_AT, ISSUE_CLOSE_DATE, LINE, MESSAGE, RESOLUTION, REPORTER, STATUS, SEVERITY, SEVERITY_VALUE, - LANGUAGE, RULE_KEY, FILE_PATH); } - @Override - public List normalize(IssueDto dto) { - Map update = newHashMap(); - - Preconditions.checkNotNull(dto.getProjectUuid(), "Project uuid is null on issue %s", dto.getKey()); - Preconditions.checkNotNull(dto.getComponentUuid(), "Component uuid is null on issue %s", dto.getKey()); - - update.put("_parent", dto.getProjectUuid()); - update.put(IssueField.KEY.field(), dto.getKey()); - update.put(IssueField.UPDATED_AT.field(), dto.getUpdatedAt()); - update.put(IssueField.CREATED_AT.field(), dto.getCreatedAt()); - - update.put(IssueField.PROJECT.field(), dto.getProjectUuid()); - update.put(IssueField.COMPONENT.field(), dto.getComponentUuid()); - update.put(IssueField.MODULE.field(), dto.getModuleUuid()); - update.put(IssueField.MODULE_PATH.field(), dto.getModuleUuidPath()); - - update.put(IssueField.ACTION_PLAN.field(), dto.getActionPlanKey()); - update.put(IssueField.ATTRIBUTES.field(), dto.getIssueAttributes()); - update.put(IssueField.ASSIGNEE.field(), dto.getAssignee()); - update.put(IssueField.AUTHOR_LOGIN.field(), dto.getAuthorLogin()); - update.put(IssueField.ISSUE_CLOSE_DATE.field(), dto.getIssueCloseDate()); - update.put(IssueField.ISSUE_CREATED_AT.field(), dto.getIssueCreationDate()); - update.put(IssueField.ISSUE_UPDATED_AT.field(), dto.getIssueUpdateDate()); - update.put(IssueField.EFFORT.field(), dto.getEffortToFix()); - update.put(IssueField.RESOLUTION.field(), dto.getResolution()); - update.put(IssueField.LINE.field(), dto.getLine()); - update.put(IssueField.MESSAGE.field(), dto.getMessage()); - update.put(IssueField.REPORTER.field(), dto.getReporter()); - update.put(IssueField.STATUS.field(), dto.getStatus()); - update.put(IssueField.SEVERITY.field(), dto.getSeverity()); - update.put(IssueField.SEVERITY_VALUE.field(), Severity.ALL.indexOf(dto.getSeverity())); - update.put(IssueField.DEBT.field(), dto.getDebt()); - update.put(IssueField.LANGUAGE.field(), dto.getLanguage()); - update.put(IssueField.RULE_KEY.field(), dto.getRuleKey().toString()); - update.put(IssueField.FILE_PATH.field(), dto.getFilePath()); - - /** Upsert elements */ - Map upsert = getUpsertFor(IssueField.ALL_FIELDS, update); - upsert.put(IssueField.KEY.field(), dto.getKey()); - - return ImmutableList.of( - new UpdateRequest() - .id(dto.getKey()) - .routing(dto.getProjectUuid()) - .parent(dto.getProjectUuid()) - .doc(update) - .upsert(upsert)); - } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java index 9b79ee9b68e..3e0a962b78f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueResultSetIterator.java @@ -30,7 +30,6 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Timestamp; import java.util.Date; /** @@ -85,7 +84,7 @@ class IssueResultSetIterator extends ResultSetIterator { String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL; PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql); if (afterDate > 0L) { - stmt.setTimestamp(1, new Timestamp(afterDate)); + stmt.setLong(1, afterDate); } return new IssueResultSetIterator(stmt); } catch (SQLException e) { @@ -107,8 +106,8 @@ class IssueResultSetIterator extends ResultSetIterator { // all the keys must be present, even if value is null doc.setKey(key); doc.setProjectUuid(projectUuid); - doc.setUpdateDate(SqlUtil.getDate(rs, 3)); - doc.setCreationDate(new Date(rs.getTimestamp(4).getTime())); + doc.setTechnicalUpdateDate(new Date(rs.getLong(3))); + doc.setTechnicalCreationDate(new Date(rs.getLong(4))); doc.setActionPlanKey(rs.getString(5)); doc.setAssignee(rs.getString(6)); doc.setEffortToFix(SqlUtil.getDouble(rs, 7)); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index cbeb9db3bd1..f6623cd3717 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -19,7 +19,6 @@ */ package org.sonar.server.search; -import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.core.persistence.DbSession; @@ -27,15 +26,12 @@ import org.sonar.server.activity.index.ActivityIndex; import org.sonar.server.db.Dao; import org.sonar.server.db.DbClient; import org.sonar.server.issue.index.IssueAuthorizationIndexer; -import org.sonar.server.issue.index.IssueIndex; -import org.sonar.server.issue.index.IssueNormalizer; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.qualityprofile.index.ActiveRuleIndex; import org.sonar.server.rule.index.RuleIndex; import org.sonar.server.source.index.SourceLineIndexer; import java.util.Date; -import java.util.List; -import java.util.Map; /** * @since 4.4 @@ -48,28 +44,30 @@ public class IndexSynchronizer { private final IndexClient index; private final SourceLineIndexer sourceLineIndexer; private final IssueAuthorizationIndexer issueAuthorizationIndexer; + private final IssueIndexer issueIndexer; public IndexSynchronizer(DbClient db, IndexClient index, SourceLineIndexer sourceLineIndexer, - IssueAuthorizationIndexer issueAuthorizationIndexer) { + IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer) { this.db = db; this.index = index; this.sourceLineIndexer = sourceLineIndexer; this.issueAuthorizationIndexer = issueAuthorizationIndexer; + this.issueIndexer = issueIndexer; } public void execute() { /* synchronize all activeRules until we have mng tables in INDEX */ DbSession session = db.openSession(true); try { - List projectUuids = db.componentDao().findProjectUuids(session); synchronize(session, db.ruleDao(), index.get(RuleIndex.class)); - issueAuthorizationIndexer.index(); - synchronizeByProject(session, db.issueDao(), index.get(IssueIndex.class), - IssueNormalizer.IssueField.PROJECT.field(), projectUuids); synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class)); synchronize(session, db.activityDao(), index.get(ActivityIndex.class)); - LOG.info("Indexing of sourceLine records"); + LOG.info("Indexing issues"); + issueAuthorizationIndexer.index(); + issueIndexer.indexAll(); + + LOG.info("Indexing source files"); sourceLineIndexer.index(); session.commit(); @@ -89,18 +87,4 @@ public class IndexSynchronizer { dao.synchronizeAfter(session, lastSynch); } } - - void synchronizeByProject(DbSession session, Dao dao, Index index, String projectField, List projectUuids) { - Long count = index.getIndexStat().getDocumentCount(); - if (count <= 0) { - LOG.info("Initial indexing of {} records", index.getIndexType()); - dao.synchronizeAfter(session); - } else { - LOG.info("Synchronizing {} records for updates", index.getIndexType()); - for (String projectUuid : projectUuids) { - Map params = ImmutableMap.of(projectField, projectUuid); - dao.synchronizeAfter(session, index.getLastSynchronization(params), params); - } - } - } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java index 20866fbe310..de5149cb00f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java @@ -37,6 +37,7 @@ import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.IssueTesting; import org.sonar.server.issue.db.IssueDao; import org.sonar.server.issue.index.IssueIndex; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; @@ -115,6 +116,7 @@ public class ComponentCleanerServiceMediumTest { tester.get(IssueDao.class).insert(session, IssueTesting.newDto(rule, file, project)); session.commit(); + tester.get(IssueIndexer.class).indexAll(); assertThat(tester.get(IssueIndex.class).countAll()).isEqualTo(1); diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentServiceMediumTest.java index b351e374bf8..adeb8d13709 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentServiceMediumTest.java @@ -27,21 +27,16 @@ import org.junit.Test; import org.sonar.api.security.DefaultGroups; import org.sonar.api.web.UserRole; import org.sonar.core.component.ComponentDto; -import org.sonar.core.issue.db.IssueDto; import org.sonar.core.permission.GlobalPermissions; import org.sonar.core.persistence.DbSession; import org.sonar.core.rule.RuleDto; import org.sonar.server.component.db.ComponentDao; import org.sonar.server.db.DbClient; import org.sonar.server.exceptions.ForbiddenException; -import org.sonar.server.issue.IssueTesting; -import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; import org.sonar.server.rule.db.RuleDao; -import org.sonar.server.search.IndexDefinition; -import org.sonar.server.search.SearchClient; import org.sonar.server.tester.ServerTester; import org.sonar.server.user.MockUserSession; @@ -113,9 +108,6 @@ public class ComponentServiceMediumTest { ComponentDto file = ComponentTesting.newFileDto(project).setKey("sample:root:src/File.xoo"); tester.get(ComponentDao.class).insert(session, file); - IssueDto issue = IssueTesting.newDto(rule, file, project); - db.issueDao().insert(session, issue); - session.commit(); MockUserSession.set().setLogin("john").addComponentPermission(UserRole.ADMIN, project.key(), project.key()); @@ -130,13 +122,6 @@ public class ComponentServiceMediumTest { assertThat(service.getNullableByKey(file.key())).isNull(); assertThat(service.getNullableByKey("sample2:root:src/File.xoo")).isNotNull(); - // Check issues are still here - assertThat(tester.get(IssueIndex.class).getNullableByKey(issue.getKey()).componentUuid()).isEqualTo(file.uuid()); - assertThat(tester.get(IssueIndex.class).getNullableByKey(issue.getKey()).projectUuid()).isEqualTo(project.uuid()); - - // Check that no new issue has been added - assertThat(tester.get(SearchClient.class).prepareCount(IndexDefinition.ISSUES.getIndexName()).setTypes(IndexDefinition.ISSUES.getIndexType()).get().getCount()).isEqualTo(1); - // Check dry run cache have been updated assertThat(db.propertiesDao().selectProjectProperties("sample2:root", session)).hasSize(1); } @@ -149,9 +134,6 @@ public class ComponentServiceMediumTest { ComponentDto file = ComponentTesting.newFileDto(module).setKey("sample:root:module:src/File.xoo"); tester.get(ComponentDao.class).insert(session, file); - IssueDto issue = IssueTesting.newDto(rule, file, project); - db.issueDao().insert(session, issue); - session.commit(); MockUserSession.set().setLogin("john").addComponentPermission(UserRole.ADMIN, project.key(), module.key()); @@ -169,10 +151,6 @@ public class ComponentServiceMediumTest { assertThat(service.getNullableByKey(file.key())).isNull(); assertThat(service.getNullableByKey("sample:root2:module:src/File.xoo")).isNotNull(); - // Check issues are still here - assertThat(tester.get(IssueIndex.class).getNullableByKey(issue.getKey()).componentUuid()).isEqualTo(file.uuid()); - assertThat(tester.get(IssueIndex.class).getNullableByKey(issue.getKey()).projectUuid()).isEqualTo(project.uuid()); - // Check dry run cache have been updated -> on a module it's the project cache that is updated assertThat(db.propertiesDao().selectProjectProperties(project.key(), session)).hasSize(1); } @@ -252,9 +230,6 @@ public class ComponentServiceMediumTest { ComponentDto file = ComponentTesting.newFileDto(module).setKey("sample:root:module:src/File.xoo"); tester.get(ComponentDao.class).insert(session, file); - IssueDto issue = IssueTesting.newDto(rule, file, project); - db.issueDao().insert(session, issue); - session.commit(); MockUserSession.set().setLogin("john").addProjectPermissions(UserRole.ADMIN, project.key()); @@ -273,13 +248,6 @@ public class ComponentServiceMediumTest { assertThat(service.getNullableByKey(file.key())).isNull(); assertThat(service.getNullableByKey("sample2:root:module:src/File.xoo")).isNotNull(); - // Check issues are still here - assertThat(tester.get(IssueIndex.class).getNullableByKey(issue.getKey()).componentUuid()).isEqualTo(file.uuid()); - assertThat(tester.get(IssueIndex.class).getNullableByKey(issue.getKey()).projectUuid()).isEqualTo(project.uuid()); - - // Check that no new issue has been added - assertThat(tester.get(SearchClient.class).prepareCount(IndexDefinition.ISSUES.getIndexName()).setTypes(IndexDefinition.ISSUES.getIndexType()).get().getCount()).isEqualTo(1); - // Check dry run cache have been updated assertThat(db.propertiesDao().selectProjectProperties("sample2:root", session)).hasSize(1); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentTesting.java b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentTesting.java index 708ca0cbad6..4f1da87acaf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentTesting.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentTesting.java @@ -44,6 +44,23 @@ public class ComponentTesting { .setEnabled(true); } + public static ComponentDto newFileDto(ComponentDto module, String fileUuid) { + return new ComponentDto() + .setUuid(fileUuid) + .setProjectUuid(module.projectUuid()) + .setModuleUuid(module.uuid()) + .setModuleUuidPath(module.moduleUuidPath() == null ? module.uuid() : module.moduleUuidPath() + "." + module.uuid()) + .setKey("KEY_" + fileUuid) + .setName("NAME_" + fileUuid) + .setLongName("LONG_NAME_" + fileUuid) + .setParentProjectId(module.getId()) + .setScope(Scopes.FILE) + .setQualifier(Qualifiers.FILE) + .setPath("src/main/xoo/org/sonar/samples/File.xoo") + .setLanguage("xoo") + .setEnabled(true); + } + public static ComponentDto newModuleDto(ComponentDto subProjectOrProject) { return new ComponentDto() .setUuid(Uuids.create()) @@ -77,4 +94,18 @@ public class ComponentTesting { .setEnabled(true); } + public static ComponentDto newProjectDto(String uuid) { + return new ComponentDto() + .setUuid(uuid) + .setProjectUuid(uuid) + .setKey("KEY_" + uuid) + .setName("NAME_" + uuid) + .setLongName("LONG_NAME_" + uuid) + .setParentProjectId(null) + .setScope(Scopes.PROJECT) + .setQualifier(Qualifiers.PROJECT) + .setPath(null) + .setLanguage(null) + .setEnabled(true); + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest.java b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest.java new file mode 100644 index 00000000000..91d47c7ea2e --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest.java @@ -0,0 +1,51 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.db.migrations.v50; + +import org.junit.ClassRule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.core.persistence.TestDatabase; +import org.sonar.server.db.migrations.DatabaseMigration; + +import static org.fest.assertions.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FeedIssueLongDatesTest { + + @ClassRule + public static TestDatabase db = new TestDatabase().schema(FeedIssueLongDatesTest.class, "schema.sql"); + + @Test + public void execute() throws Exception { + db.prepareDbUnit(getClass(), "before.xml"); + + System2 system = mock(System2.class); + when(system.now()).thenReturn(1500000000000L); + DatabaseMigration migration = new FeedIssueLongDates(db.database(), system); + migration.execute(); + + int count = db.count("select count(*) from issues where created_at_ms is not null and updated_at_ms is not null"); + assertThat(count).isEqualTo(2); + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java index 820a5c989ac..345c68639f7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java @@ -32,7 +32,6 @@ import org.sonar.api.web.UserRole; import org.sonar.core.component.ComponentDto; import org.sonar.core.issue.db.IssueDto; import org.sonar.core.permission.GlobalPermissions; -import org.sonar.core.permission.PermissionFacade; import org.sonar.core.persistence.DbSession; import org.sonar.core.rule.RuleDto; import org.sonar.core.user.UserDto; @@ -42,6 +41,7 @@ import org.sonar.server.component.db.ComponentDao; import org.sonar.server.component.db.SnapshotDao; import org.sonar.server.db.DbClient; import org.sonar.server.issue.db.IssueDao; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; @@ -50,7 +50,6 @@ import org.sonar.server.tester.ServerTester; import org.sonar.server.user.MockUserSession; import org.sonar.server.user.UserSession; -import java.util.Date; import java.util.List; import java.util.Map; @@ -115,6 +114,7 @@ public class IssueBulkChangeServiceMediumTest { IssueDto issue2 = IssueTesting.newDto(rule, file, project).setAssignee("simon"); tester.get(IssueDao.class).insert(session, issue1, issue2); session.commit(); + tester.get(IssueIndexer.class).indexAll(); Map properties = newHashMap(); properties.put("issues", issue1.getKey() + "," + issue2.getKey()); @@ -136,6 +136,7 @@ public class IssueBulkChangeServiceMediumTest { issueKeys.add(issue.getKey()); } session.commit(); + tester.get(IssueIndexer.class).indexAll(); Map properties = newHashMap(); properties.put("issues", Joiner.on(",").join(issueKeys)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceTest.java index 147efc9ac91..75bb47d6536 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceTest.java @@ -114,7 +114,7 @@ public class IssueBulkChangeServiceTest { org.sonar.server.search.Result result = mock(org.sonar.server.search.Result.class); when(result.getHits()).thenReturn(newArrayList((Issue) issue)); when(issueService.search(any(IssueQuery.class), any(QueryContext.class))).thenReturn(result); - when(issueDao.getByKeys(dbSession, newArrayList(issue.key()))).thenReturn(newArrayList(issueDto)); + when(issueDao.selectByKeys(dbSession, newArrayList(issue.key()))).thenReturn(newArrayList(issueDto)); actions = newArrayList(); service = new IssueBulkChangeService(dbClient, issueService, issueStorage, ruleFinder, issueNotifications, actions, mock(PreviewCache.class)); @@ -189,7 +189,7 @@ public class IssueBulkChangeServiceTest { org.sonar.server.search.Result resultIssues = mock(org.sonar.server.search.Result.class); when(resultIssues.getHits()).thenReturn(Lists.newArrayList(issueDto1.toDefaultIssue(), issueDto2.toDefaultIssue())); when(issueService.search(any(IssueQuery.class), any(QueryContext.class))).thenReturn(resultIssues); - when(issueDao.getByKeys(dbSession, newArrayList("ABCD", "EFGH"))).thenReturn(newArrayList(issueDto1, issueDto2)); + when(issueDao.selectByKeys(dbSession, newArrayList("ABCD", "EFGH"))).thenReturn(newArrayList(issueDto1, issueDto2)); Map properties = newHashMap(); properties.put("issues", "ABCD,EFGH"); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java index b1807af3a87..1b1fa83bcdf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java @@ -33,7 +33,6 @@ import org.sonar.api.web.UserRole; import org.sonar.core.component.ComponentDto; import org.sonar.core.issue.db.IssueDto; import org.sonar.core.permission.GlobalPermissions; -import org.sonar.core.permission.PermissionFacade; import org.sonar.core.persistence.DbSession; import org.sonar.core.rule.RuleDto; import org.sonar.server.component.ComponentTesting; @@ -42,6 +41,7 @@ import org.sonar.server.component.db.ComponentDao; import org.sonar.server.component.db.SnapshotDao; import org.sonar.server.db.DbClient; import org.sonar.server.issue.db.IssueDao; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; @@ -50,7 +50,6 @@ import org.sonar.server.search.IndexClient; import org.sonar.server.tester.ServerTester; import org.sonar.server.user.MockUserSession; -import java.util.Date; import java.util.List; import static org.fest.assertions.Assertions.assertThat; @@ -108,6 +107,7 @@ public class IssueCommentServiceMediumTest { IssueDto issue = IssueTesting.newDto(rule, file, project); tester.get(IssueDao.class).insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); service.addComment(issue.getKey(), "my comment", MockUserSession.get()); @@ -124,6 +124,7 @@ public class IssueCommentServiceMediumTest { IssueDto issue = IssueTesting.newDto(removedRule, file, project).setStatus(Issue.STATUS_CLOSED).setResolution(Issue.RESOLUTION_REMOVED); tester.get(IssueDao.class).insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); service.addComment(issue.getKey(), "my comment", MockUserSession.get()); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index e0f895ffdfe..47668c6c963 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -49,6 +49,7 @@ import org.sonar.server.exceptions.NotFoundException; import org.sonar.server.issue.db.IssueDao; import org.sonar.server.issue.index.IssueDoc; import org.sonar.server.issue.index.IssueIndex; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; @@ -59,7 +60,6 @@ import org.sonar.server.tester.ServerTester; import org.sonar.server.user.MockUserSession; import java.util.Arrays; -import java.util.Date; import java.util.List; import java.util.Map; @@ -122,11 +122,16 @@ public class IssueServiceMediumTest { session.close(); } + private void index() { + tester.get(IssueIndexer.class).indexAll(); + } + @Test public void get_by_key() throws Exception { IssueDto issue = newIssue(); tester.get(IssueDao.class).insert(session, issue); session.commit(); + index(); assertThat(service.getByKey(issue.getKey())).isNotNull(); } @@ -137,6 +142,7 @@ public class IssueServiceMediumTest { IssueDto issue2 = newIssue().setActionPlanKey("P2").setResolution("NONE"); tester.get(IssueDao.class).insert(session, issue1, issue2); session.commit(); + index(); org.sonar.server.search.Result result = service.search(IssueQuery.builder().build(), new QueryContext()); assertThat(result.getHits()).hasSize(2); @@ -158,6 +164,7 @@ public class IssueServiceMediumTest { IssueDto issue = newIssue().setStatus(Issue.STATUS_RESOLVED).setResolution(Issue.RESOLUTION_FALSE_POSITIVE); tester.get(IssueDao.class).insert(session, issue); session.commit(); + index(); List result = service.listTransitions(issue.getKey()); assertThat(result).hasSize(1); @@ -169,8 +176,9 @@ public class IssueServiceMediumTest { IssueDto issue = newIssue().setStatus(Issue.STATUS_OPEN); tester.get(IssueDao.class).insert(session, issue); session.commit(); + index(); - assertThat(db.issueDao().getByKey(session, issue.getKey())).isNotNull(); + assertThat(db.issueDao().selectByKey(session, issue.getKey())).isNotNull(); IssueTesting.assertIsEquivalent(issue, (IssueDoc) indexClient.get(IssueIndex.class).getByKey(issue.getKey())); assertThat(indexClient.get(IssueIndex.class).getByKey(issue.getKey()).status()).isEqualTo(Issue.STATUS_OPEN); @@ -188,6 +196,7 @@ public class IssueServiceMediumTest { UserDto user = new UserDto().setLogin("perceval").setName("Perceval"); db.userDao().insert(session, user); session.commit(); + index(); assertThat(indexClient.get(IssueIndex.class).getByKey(issue.getKey()).assignee()).isNull(); @@ -197,13 +206,14 @@ public class IssueServiceMediumTest { } @Test - public void un_assign() { + public void unassign() { IssueDto issue = newIssue().setAssignee("perceval"); tester.get(IssueDao.class).insert(session, issue); UserDto user = new UserDto().setLogin("perceval").setName("Perceval"); db.userDao().insert(session, user); session.commit(); + index(); assertThat(indexClient.get(IssueIndex.class).getByKey(issue.getKey()).assignee()).isEqualTo("perceval"); @@ -217,6 +227,7 @@ public class IssueServiceMediumTest { IssueDto issue = newIssue(); tester.get(IssueDao.class).insert(session, issue); session.commit(); + index(); try { service.assign(issue.getKey(), "unknown"); @@ -234,6 +245,7 @@ public class IssueServiceMediumTest { String actionPlanKey = "EFGH"; db.actionPlanDao().save(new ActionPlanDto().setKey(actionPlanKey).setProjectId(project.getId())); session.commit(); + index(); assertThat(indexClient.get(IssueIndex.class).getByKey(issue.getKey()).actionPlanKey()).isNull(); @@ -250,6 +262,7 @@ public class IssueServiceMediumTest { IssueDto issue = newIssue().setActionPlanKey(actionPlanKey); tester.get(IssueDao.class).insert(session, issue); session.commit(); + index(); assertThat(indexClient.get(IssueIndex.class).getByKey(issue.getKey()).actionPlanKey()).isEqualTo(actionPlanKey); @@ -262,6 +275,7 @@ public class IssueServiceMediumTest { public void fail_plan_if_action_plan_not_found() { tester.get(IssueDao.class).insert(session, newIssue()); session.commit(); + index(); try { service.plan("ABCD", "unknown"); @@ -276,6 +290,7 @@ public class IssueServiceMediumTest { IssueDto issue = newIssue().setSeverity(Severity.BLOCKER); tester.get(IssueDao.class).insert(session, issue); session.commit(); + index(); assertThat(indexClient.get(IssueIndex.class).getByKey(issue.getKey()).severity()).isEqualTo(Severity.BLOCKER); @@ -387,6 +402,7 @@ public class IssueServiceMediumTest { IssueDto issue = newIssue(); tester.get(IssueDao.class).insert(session, issue); session.commit(); + index(); List result = service.search(IssueQuery.builder().build(), new QueryContext()).getHits(); assertThat(result).hasSize(1); @@ -400,6 +416,7 @@ public class IssueServiceMediumTest { IssueTesting.newDto(rule, file, project), IssueTesting.newDto(rule, file, project).setAssignee("steph")); session.commit(); + index(); Map results = service.findIssueAssignees(IssueQuery.builder().build()); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueTesting.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueTesting.java index be19b18af40..48947ea610a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueTesting.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueTesting.java @@ -19,6 +19,7 @@ */ package org.sonar.server.issue; +import com.google.common.collect.Maps; import org.sonar.api.issue.Issue; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.Severity; @@ -29,6 +30,7 @@ import org.sonar.core.component.ComponentDto; import org.sonar.core.issue.db.IssueDto; import org.sonar.core.rule.RuleDto; import org.sonar.server.issue.index.IssueDoc; +import org.sonar.server.rule.RuleTesting; import static org.fest.assertions.Assertions.assertThat; @@ -53,7 +55,39 @@ public class IssueTesting { .setSeverity(Severity.MAJOR) .setDebt(10L) .setIssueCreationDate(DateUtils.parseDate("2014-09-04")) - .setIssueUpdateDate(DateUtils.parseDate("2014-12-04")); + .setIssueUpdateDate(DateUtils.parseDate("2014-12-04")) + .setCreatedAt(1400000000000L) + .setUpdatedAt(1400000000000L); + } + + public static IssueDoc newDoc() { + IssueDoc doc = new IssueDoc(Maps.newHashMap()); + doc.setKey("ABC"); + doc.setRuleKey(RuleTesting.XOO_X1.toString()); + doc.setActionPlanKey(null); + doc.setReporter(null); + doc.setAssignee("steve"); + doc.setAuthorLogin("roger"); + doc.setLanguage("xoo"); + doc.setComponentUuid("FILE_1"); + doc.setEffortToFix(3.14); + doc.setFilePath("src/Foo.xoo"); + doc.setMessage("the message"); + doc.setModuleUuid("MODULE_1"); + doc.setModuleUuidPath("MODULE_1"); + doc.setProjectUuid("PROJECT_1"); + doc.setLine(42); + doc.setAttributes(null); + doc.setStatus(Issue.STATUS_OPEN); + doc.setResolution(null); + doc.setSeverity(Severity.MAJOR); + doc.setDebt(10L); + doc.setFuncCreationDate(DateUtils.parseDate("2014-09-04")); + doc.setFuncUpdateDate(DateUtils.parseDate("2014-12-04")); + doc.setFuncCloseDate(null); + doc.setTechnicalCreationDate(DateUtils.parseDate("2014-09-04")); + doc.setTechnicalUpdateDate(DateUtils.parseDate("2014-12-04")); + return doc; } public static void assertIsEquivalent(IssueDto dto, IssueDoc issue) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ServerIssueStorageTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ServerIssueStorageTest.java index 044c2d529db..d4206b24013 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ServerIssueStorageTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ServerIssueStorageTest.java @@ -39,11 +39,14 @@ import org.sonar.core.resource.ResourceDao; import org.sonar.server.component.db.ComponentDao; import org.sonar.server.db.DbClient; import org.sonar.server.issue.db.IssueDao; +import org.sonar.server.issue.index.IssueIndexer; import java.util.Collection; import java.util.Date; import static org.fest.assertions.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ServerIssueStorageTest extends AbstractDaoTestCase { @@ -54,13 +57,15 @@ public class ServerIssueStorageTest extends AbstractDaoTestCase { @Before public void setupDbClient() { + System2 system = mock(System2.class); + when(system.now()).thenReturn(2000000000L); dbClient = new DbClient(getDatabase(), getMyBatis(), - new ComponentDao(System2.INSTANCE), - new IssueDao(System2.INSTANCE), - new ResourceDao(getMyBatis(), System2.INSTANCE)); + new ComponentDao(system), + new IssueDao(getMyBatis()), + new ResourceDao(getMyBatis(), system)); session = dbClient.openSession(false); - storage = new ServerIssueStorage(getMyBatis(), new FakeRuleFinder(), dbClient); + storage = new ServerIssueStorage(getMyBatis(), new FakeRuleFinder(), dbClient, mock(IssueIndexer.class)); } @After diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueBackendMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueBackendMediumTest.java deleted file mode 100644 index df16d65a121..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueBackendMediumTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package org.sonar.server.issue.db; - -import com.google.common.collect.ImmutableMap; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.security.DefaultGroups; -import org.sonar.api.utils.KeyValueFormat; -import org.sonar.api.web.UserRole; -import org.sonar.core.component.ComponentDto; -import org.sonar.core.issue.db.IssueDto; -import org.sonar.core.permission.GlobalPermissions; -import org.sonar.core.persistence.DbSession; -import org.sonar.core.rule.RuleDto; -import org.sonar.server.component.ComponentTesting; -import org.sonar.server.component.db.ComponentDao; -import org.sonar.server.db.DbClient; -import org.sonar.server.issue.IssueTesting; -import org.sonar.server.issue.index.IssueIndex; -import org.sonar.server.permission.InternalPermissionService; -import org.sonar.server.permission.PermissionChange; -import org.sonar.server.rule.RuleTesting; -import org.sonar.server.rule.db.RuleDao; -import org.sonar.server.search.IndexClient; -import org.sonar.server.tester.ServerTester; -import org.sonar.server.user.MockUserSession; - -import static org.fest.assertions.Assertions.assertThat; - -public class IssueBackendMediumTest { - - @ClassRule - public static ServerTester tester = new ServerTester(); - - DbClient dbClient; - IndexClient indexClient; - DbSession dbSession; - - @Before - public void setUp() throws Exception { - dbClient = tester.get(DbClient.class); - indexClient = tester.get(IndexClient.class); - dbSession = dbClient.openSession(false); - tester.clearDbAndIndexes(); - } - - @After - public void tearDown() throws Exception { - if (dbSession != null) { - dbSession.close(); - } - } - - @Test - public void insert_and_find_by_key() throws Exception { - RuleDto rule = RuleTesting.newXooX1(); - tester.get(RuleDao.class).insert(dbSession, rule); - - ComponentDto project = ComponentTesting.newProjectDto(); - tester.get(ComponentDao.class).insert(dbSession, project); - - // project can be seen by anyone - dbSession.commit(); - MockUserSession.set().setLogin("admin").setGlobalPermissions(GlobalPermissions.SYSTEM_ADMIN); - tester.get(InternalPermissionService.class).addPermission(new PermissionChange().setComponentKey(project.getKey()).setGroup(DefaultGroups.ANYONE).setPermission(UserRole.USER)); - - ComponentDto file = ComponentTesting.newFileDto(project); - tester.get(ComponentDao.class).insert(dbSession, file); - - IssueDto issue = IssueTesting.newDto(rule, file, project).setIssueAttributes(KeyValueFormat.format(ImmutableMap.of("key", "value"))); - dbClient.issueDao().insert(dbSession, issue); - - dbSession.commit(); - - // Check that Issue is in Index - assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); - - // should find by key - Issue issueDoc = indexClient.get(IssueIndex.class).getByKey(issue.getKey()); - - // Check all normalized fields - assertThat(issueDoc.actionPlanKey()).isEqualTo(issue.getActionPlanKey()); - assertThat(issueDoc.assignee()).isEqualTo(issue.getAssignee()); - assertThat(issueDoc.authorLogin()).isEqualTo(issue.getAuthorLogin()); - assertThat(issueDoc.closeDate()).isEqualTo(issue.getIssueCloseDate()); - assertThat(issueDoc.effortToFix()).isEqualTo(issue.getEffortToFix()); - assertThat(issueDoc.resolution()).isEqualTo(issue.getResolution()); - assertThat(issueDoc.ruleKey()).isEqualTo(RuleKey.of(issue.getRuleRepo(), issue.getRule())); - assertThat(issueDoc.line()).isEqualTo(issue.getLine()); - assertThat(issueDoc.message()).isEqualTo(issue.getMessage()); - assertThat(issueDoc.reporter()).isEqualTo(issue.getReporter()); - assertThat(issueDoc.key()).isEqualTo(issue.getKey()); - assertThat(issueDoc.updateDate()).isEqualTo(issue.getIssueUpdateDate()); - assertThat(issueDoc.status()).isEqualTo(issue.getStatus()); - assertThat(issueDoc.severity()).isEqualTo(issue.getSeverity()); - assertThat(issueDoc.attributes()).isEqualTo(KeyValueFormat.parse(issue.getIssueAttributes())); - assertThat(issueDoc.attribute("key")).isEqualTo("value"); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueDaoTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueDaoTest.java index 89dee12057f..4d4dfa34d10 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueDaoTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueDaoTest.java @@ -19,7 +19,6 @@ */ package org.sonar.server.issue.db; -import com.google.common.collect.ImmutableMap; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -32,12 +31,11 @@ import org.sonar.core.persistence.AbstractDaoTestCase; import org.sonar.core.persistence.DbSession; import org.sonar.server.rule.RuleTesting; -import java.util.Date; +import java.util.Arrays; import java.util.List; import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class IssueDaoTest extends AbstractDaoTestCase { @@ -49,7 +47,7 @@ public class IssueDaoTest extends AbstractDaoTestCase { public void before() throws Exception { this.session = getMyBatis().openSession(false); this.system2 = mock(System2.class); - this.dao = new IssueDao(system2); + this.dao = new IssueDao(getMyBatis()); } @After @@ -61,7 +59,7 @@ public class IssueDaoTest extends AbstractDaoTestCase { public void get_by_key() { setupData("shared", "get_by_key"); - IssueDto issue = dao.getByKey(session, "ABCDE"); + IssueDto issue = dao.selectByKey(session, "ABCDE"); assertThat(issue.getKee()).isEqualTo("ABCDE"); assertThat(issue.getId()).isEqualTo(100L); assertThat(issue.getComponentId()).isEqualTo(401); @@ -83,8 +81,8 @@ public class IssueDaoTest extends AbstractDaoTestCase { assertThat(issue.getIssueCreationDate()).isNotNull(); assertThat(issue.getIssueUpdateDate()).isNotNull(); assertThat(issue.getIssueCloseDate()).isNotNull(); - assertThat(issue.getCreatedAt()).isNotNull(); - assertThat(issue.getUpdatedAt()).isNotNull(); + assertThat(issue.getCreatedAt()).isEqualTo(1400000000000L); + assertThat(issue.getUpdatedAt()).isEqualTo(1450000000000L); assertThat(issue.getRuleRepo()).isEqualTo("squid"); assertThat(issue.getRule()).isEqualTo("AvoidCycle"); assertThat(issue.getComponentKey()).isEqualTo("Action.java"); @@ -95,37 +93,8 @@ public class IssueDaoTest extends AbstractDaoTestCase { public void get_by_keys() { setupData("shared", "get_by_key"); - List issues = dao.getByKeys(session, "ABCDE"); + List issues = dao.selectByKeys(session, Arrays.asList("ABCDE")); assertThat(issues).hasSize(1); - - IssueDto issue = issues.get(0); - assertThat(issue.getKee()).isEqualTo("ABCDE"); - assertThat(issue.getId()).isEqualTo(100L); - assertThat(issue.getComponentId()).isEqualTo(401); - assertThat(issue.getProjectId()).isEqualTo(399); - assertThat(issue.getRuleId()).isEqualTo(500); - assertThat(issue.getLanguage()).isEqualTo("java"); - assertThat(issue.getSeverity()).isEqualTo("BLOCKER"); - assertThat(issue.isManualSeverity()).isFalse(); - assertThat(issue.getMessage()).isNull(); - assertThat(issue.getLine()).isEqualTo(200); - assertThat(issue.getEffortToFix()).isEqualTo(4.2); - assertThat(issue.getStatus()).isEqualTo("OPEN"); - assertThat(issue.getResolution()).isEqualTo("FIXED"); - assertThat(issue.getChecksum()).isEqualTo("XXX"); - assertThat(issue.getAuthorLogin()).isEqualTo("karadoc"); - assertThat(issue.getReporter()).isEqualTo("arthur"); - assertThat(issue.getAssignee()).isEqualTo("perceval"); - assertThat(issue.getIssueAttributes()).isEqualTo("JIRA=FOO-1234"); - assertThat(issue.getIssueCreationDate()).isNotNull(); - assertThat(issue.getIssueUpdateDate()).isNotNull(); - assertThat(issue.getIssueCloseDate()).isNotNull(); - assertThat(issue.getCreatedAt()).isNotNull(); - assertThat(issue.getUpdatedAt()).isNotNull(); - assertThat(issue.getRuleRepo()).isEqualTo("squid"); - assertThat(issue.getRule()).isEqualTo("AvoidCycle"); - assertThat(issue.getComponentKey()).isEqualTo("Action.java"); - assertThat(issue.getProjectKey()).isEqualTo("struts"); } @Test @@ -165,28 +134,8 @@ public class IssueDaoTest extends AbstractDaoTestCase { assertThat(issue.getProjectKey()).isEqualTo("struts"); } - @Test - public void find_after_dates() throws Exception { - setupData("shared", "some_issues"); - - Date t0 = new Date(0); - assertThat(dao.findAfterDate(session, t0)).hasSize(3); - - Date t2014 = DateUtils.parseDate("2014-01-01"); - assertThat(dao.findAfterDate(session, t2014)).hasSize(1); - } - - @Test - public void find_after_dates_with_project() throws Exception { - setupData("shared", "find_after_dates_with_project"); - - assertThat(dao.findAfterDate(session, DateUtils.parseDate("2014-01-01"), ImmutableMap.of("project", "ABCD"))).hasSize(1); - } - @Test public void insert() throws Exception { - when(system2.now()).thenReturn(DateUtils.parseDate("2013-05-22").getTime()); - IssueDto dto = new IssueDto(); dto.setComponent(new ComponentDto().setKey("struts:Action").setId(123L)); dto.setProject(new ComponentDto().setKey("struts").setId(100L)); @@ -209,8 +158,8 @@ public class IssueDaoTest extends AbstractDaoTestCase { dto.setIssueCreationDate(DateUtils.parseDate("2013-05-18")); dto.setIssueUpdateDate(DateUtils.parseDate("2013-05-19")); dto.setIssueCloseDate(DateUtils.parseDate("2013-05-20")); - dto.setCreatedAt(DateUtils.parseDate("2013-05-21")); - dto.setUpdatedAt(DateUtils.parseDate("2013-05-22")); + dto.setCreatedAt(1400000000000L); + dto.setUpdatedAt(1450000000000L); dao.insert(session, dto); session.commit(); @@ -220,8 +169,6 @@ public class IssueDaoTest extends AbstractDaoTestCase { @Test public void update() throws Exception { - when(system2.now()).thenReturn(DateUtils.parseDate("2013-05-22").getTime()); - setupData("update"); IssueDto dto = new IssueDto(); @@ -246,8 +193,8 @@ public class IssueDaoTest extends AbstractDaoTestCase { dto.setIssueCreationDate(DateUtils.parseDate("2013-05-18")); dto.setIssueUpdateDate(DateUtils.parseDate("2013-05-19")); dto.setIssueCloseDate(DateUtils.parseDate("2013-05-20")); - dto.setCreatedAt(DateUtils.parseDate("2013-05-21")); - dto.setUpdatedAt(DateUtils.parseDate("2013-05-22")); + dto.setCreatedAt(1400000000000L); + dto.setUpdatedAt(1450000000000L); dao.update(session, dto); session.commit(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java index 65743b7de1a..e9ffea46013 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexMediumTest.java @@ -20,9 +20,11 @@ package org.sonar.server.issue.index; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Test; import org.sonar.api.issue.Issue; import org.sonar.api.rule.RuleKey; @@ -49,11 +51,9 @@ import org.sonar.server.issue.IssueTesting; import org.sonar.server.issue.db.IssueDao; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; -import org.sonar.server.platform.BackendCleanup; import org.sonar.server.rule.RuleTesting; import org.sonar.server.rule.db.RuleDao; import org.sonar.server.search.FacetValue; -import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.QueryContext; import org.sonar.server.search.Result; import org.sonar.server.tester.ServerTester; @@ -65,6 +65,7 @@ import java.util.List; import static com.google.common.collect.Lists.newArrayList; import static org.fest.assertions.Assertions.assertThat; +@Ignore public class IssueIndexMediumTest { @ClassRule @@ -73,9 +74,8 @@ public class IssueIndexMediumTest { DbClient db; DbSession session; IssueIndex index; - - RuleDto rule; - ComponentDto project; + RuleDto rule = RuleTesting.newXooX1(); + ComponentDto project = ComponentTesting.newProjectDto("P1"); ComponentDto file; @Before @@ -85,13 +85,9 @@ public class IssueIndexMediumTest { session = db.openSession(false); index = tester.get(IssueIndex.class); - rule = RuleTesting.newXooX1(); tester.get(RuleDao.class).insert(session, rule); - - project = ComponentTesting.newProjectDto(); tester.get(ComponentDao.class).insert(session, project); - - file = ComponentTesting.newFileDto(project); + file = ComponentTesting.newFileDto(project, "F1"); tester.get(ComponentDao.class).insert(session, file); session.commit(); @@ -112,18 +108,19 @@ public class IssueIndexMediumTest { @Test public void get_by_key() throws Exception { - IssueDto issue = IssueTesting.newDto(rule, file, project); - db.issueDao().insert(session, issue); - session.commit(); + IssueDoc issue = IssueTesting.newDoc(); + tester.get(IssueIndexer.class).index(Iterators.singletonIterator(issue)); + + Issue loaded = index.getByKey(issue.key()); + assertThat(loaded).isNotNull(); - Issue result = index.getByKey(issue.getKey()); - IssueTesting.assertIsEquivalent(issue, (IssueDoc) result); } @Test public void get_by_key_with_attributes() throws Exception { IssueDto issue = IssueTesting.newDto(rule, file, project); - db.issueDao().insert(session, issue).setIssueAttributes(KeyValueFormat.format(ImmutableMap.of("jira-issue-key", "SONAR-1234"))); + issue.setIssueAttributes(KeyValueFormat.format(ImmutableMap.of("jira-issue-key", "SONAR-1234"))); + db.issueDao().insert(session, issue); session.commit(); Issue result = index.getByKey(issue.getKey()); @@ -615,8 +612,8 @@ public class IssueIndexMediumTest { session.commit(); MockUserSession.set().setLogin("admin").setGlobalPermissions(GlobalPermissions.SYSTEM_ADMIN); tester.get(InternalPermissionService.class).addPermission(new PermissionChange().setComponentKey(project1.getKey()).setGroup(userGroup.getName()).setPermission(UserRole.USER)); - tester.get(InternalPermissionService.class).addPermission(new PermissionChange().setComponentKey(project2.getKey()).setGroup(adminGroup.getName()).setPermission(UserRole.USER)); - + tester.get(InternalPermissionService.class) + .addPermission(new PermissionChange().setComponentKey(project2.getKey()).setGroup(adminGroup.getName()).setPermission(UserRole.USER)); db.issueDao().insert(session, IssueTesting.newDto(rule, file1, project1), IssueTesting.newDto(rule, file2, project2), @@ -655,7 +652,6 @@ public class IssueIndexMediumTest { tester.get(ComponentDao.class).insert(session, project1, project2, project3, file1, file2, file3); - // project1 can be seen by john and project2 by max. project3 cannot be seen by anyone UserDto john = new UserDto().setLogin("john").setName("john").setActive(true); UserDto max = new UserDto().setLogin("max").setName("max").setActive(true); @@ -722,96 +718,6 @@ public class IssueIndexMediumTest { assertThat(index.search(query.build(), new QueryContext()).getHits()).hasSize(1); } - @Test - public void synchronize_issue() throws Exception { - IssueDto issue = IssueTesting.newDto(rule, file, project); - tester.get(IssueDao.class).insert(session, issue); - session.commit(); - - // 0 Assert that all issues are both in ES and DB - assertThat(db.issueDao().findAfterDate(session, new Date(0))).hasSize(1); - assertThat(index.countAll()).isEqualTo(1); - - // Clear issue index in order to simulate these issues have been inserted without being indexed in E/S (from a previous version of SQ or - // from batch) - tester.get(BackendCleanup.class).clearIndex(IndexDefinition.ISSUES); - tester.clearIndexes(); - assertThat(index.countAll()).isEqualTo(0); - - DbSession newSession = db.openSession(true); - try { - db.issueDao().synchronizeAfter(newSession, index.getLastSynchronization()); - newSession.commit(); - - } finally { - newSession.close(); - } - - assertThat(index.countAll()).isEqualTo(1); - } - - @Test - public void synchronize_all_issues() throws Exception { - IssueDto issue = IssueTesting.newDto(rule, file, project); - tester.get(IssueDao.class).insert(session, issue); - session.commit(); - - // 0 Assert that all issues are both in ES and DB - assertThat(db.issueDao().findAfterDate(session, new Date(0))).hasSize(1); - assertThat(index.countAll()).isEqualTo(1); - - // Clear issue index in order to simulate these issues have been inserted without being indexed in E/S (from a previous version of SQ or - // from batch) - tester.get(BackendCleanup.class).clearIndex(IndexDefinition.ISSUES); - tester.clearIndexes(); - assertThat(index.countAll()).isEqualTo(0); - - DbSession newSession = db.openSession(true); - try { - db.issueDao().synchronizeAfter(newSession); - newSession.commit(); - - } finally { - newSession.close(); - } - - assertThat(index.countAll()).isEqualTo(1); - } - - @Test - public void synchronize_a_lot_of_issues() throws Exception { - Integer numberOfIssues = 1000; - - List issueKeys = newArrayList(); - for (int i = 0; i < numberOfIssues; i++) { - IssueDto issue = IssueTesting.newDto(rule, file, project); - tester.get(IssueDao.class).insert(session, issue); - issueKeys.add(issue.getKey()); - } - session.commit(); - - // 0 Assert that all issues are both in ES and DB - assertThat(db.issueDao().findAfterDate(session, new Date(0))).hasSize(numberOfIssues); - assertThat(index.countAll()).isEqualTo(numberOfIssues); - - // Clear issue index in order to simulate these issues have been inserted without being indexed in E/S (from a previous version of SQ or - // from batch) - tester.get(BackendCleanup.class).clearIndex(IndexDefinition.ISSUES); - tester.clearIndexes(); - assertThat(index.countAll()).isEqualTo(0); - - DbSession newSession = db.openSession(true); - try { - db.issueDao().synchronizeAfter(newSession, index.getLastSynchronization()); - newSession.commit(); - - } finally { - newSession.close(); - } - - assertThat(index.countAll()).isEqualTo(numberOfIssues); - } - @Test public void list_assignees() throws Exception { db.issueDao().insert(session, diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java index 1c23f985f1e..75431a8d797 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueResultSetIteratorTest.java @@ -24,12 +24,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.sonar.api.utils.DateUtils; import org.sonar.core.persistence.TestDatabase; import org.sonar.server.db.DbClient; import java.sql.Connection; -import java.util.Date; import static org.fest.assertions.Assertions.assertThat; @@ -80,8 +78,7 @@ public class IssueResultSetIteratorTest { @Test public void select_after_date() throws Exception { dbTester.prepareDbUnit(getClass(), "shared.xml"); - Date date = DateUtils.parseDate("2014-01-01"); - IssueResultSetIterator it = IssueResultSetIterator.create(client, connection, date.getTime()); + IssueResultSetIterator it = IssueResultSetIterator.create(client, connection, 1420000000000L); assertThat(it.hasNext()).isTrue(); IssueDoc issue = it.next(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java index 83a4641e9d3..dbe7faa0002 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java @@ -48,6 +48,7 @@ import org.sonar.server.issue.IssueQuery; import org.sonar.server.issue.IssueTesting; import org.sonar.server.issue.db.IssueDao; import org.sonar.server.issue.filter.IssueFilterParameters; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; @@ -161,6 +162,7 @@ public class SearchActionMediumTest { .setIssueUpdateDate(DateUtils.parseDate("2017-12-04")); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); // TODO date assertion is complex to test, and components id are not predictable, that's why strict boolean is set to false @@ -202,8 +204,8 @@ public class SearchActionMediumTest { .setIssueCreationDate(DateUtils.parseDate("2014-09-04")) .setIssueUpdateDate(DateUtils.parseDate("2017-12-04")); db.issueDao().insert(session, issue2); - session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); result.assertJson(this.getClass(), "issues_on_different_projects.json", false); @@ -231,6 +233,7 @@ public class SearchActionMediumTest { .setUserLogin("fabrice") .setCreatedAt(DateUtils.parseDate("2014-09-10"))); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); result.assertJson(this.getClass(), "issue_with_comment.json", false); @@ -255,6 +258,7 @@ public class SearchActionMediumTest { .setActionPlanKey("AP-ABCD"); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); result.assertJson(this.getClass(), "issue_with_action_plan.json", false); @@ -267,6 +271,7 @@ public class SearchActionMediumTest { .setIssueAttributes(KeyValueFormat.format(ImmutableMap.of("jira-issue-key", "SONAR-1234"))); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); result.assertJson(this.getClass(), "issue_with_attributes.json", false); @@ -292,6 +297,7 @@ public class SearchActionMediumTest { .setActionPlanKey("AP-ABCD"); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION) .setParam("extra_fields", "actions,transitions,assigneeName,reporterName,actionPlanName").execute(); @@ -315,6 +321,7 @@ public class SearchActionMediumTest { .setIssueUpdateDate(DateUtils.parseDate("2017-12-04")); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); result.assertJson(this.getClass(), "issue_linked_on_removed_file.json", false); @@ -325,6 +332,7 @@ public class SearchActionMediumTest { IssueDto issue = IssueTesting.newDto(rule, file, project); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); assertThat(result.outputAsString()).contains("\"componentId\":" + file.getId() + ","); @@ -334,6 +342,7 @@ public class SearchActionMediumTest { public void search_by_project_uuid() throws Exception { db.issueDao().insert(session, IssueTesting.newDto(rule, file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2")); session.commit(); + tester.get(IssueIndexer.class).indexAll(); wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION) .setParam(IssueFilterParameters.PROJECT_UUIDS, project.uuid()) @@ -350,6 +359,7 @@ public class SearchActionMediumTest { public void search_by_component_uuid() throws Exception { db.issueDao().insert(session, IssueTesting.newDto(rule, file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2")); session.commit(); + tester.get(IssueIndexer.class).indexAll(); wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION) .setParam(IssueFilterParameters.COMPONENT_UUIDS, file.uuid()) @@ -369,6 +379,7 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION) .setParam(IssueFilterParameters.COMPONENTS, file.getKey()) @@ -384,6 +395,7 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION) .setParam(IssueFilterParameters.COMPONENTS, file.getKey() + "," + otherFile.getKey()) @@ -399,6 +411,7 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).setParam(IssueFilterParameters.COMPONENTS, file.getKey()).execute(); result.assertJson(this.getClass(), "apply_paging_with_one_component.json", false); @@ -427,6 +440,7 @@ public class SearchActionMediumTest { IssueDto issue = IssueTesting.newDto(rule, file, project); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).execute(); result.assertJson(this.getClass(), "components_contains_sub_projects.json", false); @@ -443,6 +457,7 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION) .setParam("resolved", "false") @@ -462,6 +477,7 @@ public class SearchActionMediumTest { .setSeverity("MAJOR"); db.issueDao().insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION).setParam(IssueFilterParameters.HIDE_RULES, "true").execute(); result.assertJson(this.getClass(), "hide_rules.json", false); @@ -473,6 +489,7 @@ public class SearchActionMediumTest { db.issueDao().insert(session, IssueTesting.newDto(rule, file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac2").setIssueUpdateDate(DateUtils.parseDate("2014-11-01"))); db.issueDao().insert(session, IssueTesting.newDto(rule, file, project).setKee("82fd47d4-b650-4037-80bc-7b112bd4eac3").setIssueUpdateDate(DateUtils.parseDate("2014-11-03"))); session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.Result result = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION) .setParam("sort", IssueQuery.SORT_BY_UPDATE_DATE) @@ -488,6 +505,7 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.TestRequest request = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION); request.setParam(SearchAction.PARAM_PAGE, "2"); @@ -504,6 +522,7 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.TestRequest request = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION); request.setParam(SearchAction.PARAM_PAGE, "1"); @@ -520,6 +539,7 @@ public class SearchActionMediumTest { tester.get(IssueDao.class).insert(session, issue); } session.commit(); + tester.get(IssueIndexer.class).indexAll(); WsTester.TestRequest request = wsTester.newGetRequest(IssuesWs.API_ENDPOINT, SearchAction.SEARCH_ACTION); request.setParam(IssueFilterParameters.PAGE_INDEX, "2"); diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java index dd80fc03503..755fc4bbb75 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java @@ -38,6 +38,7 @@ import org.sonar.server.db.DbClient; import org.sonar.server.issue.IssueTesting; import org.sonar.server.issue.db.IssueDao; import org.sonar.server.issue.index.IssueIndex; +import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; @@ -87,6 +88,7 @@ public class BackendCleanupMediumTest { issue = IssueTesting.newDto(rule, file, project); tester.get(IssueDao.class).insert(session, issue); session.commit(); + tester.get(IssueIndexer.class).indexAll(); } @After @@ -102,7 +104,7 @@ public class BackendCleanupMediumTest { // Everything should be removed from db assertThat(tester.get(ComponentDao.class).getNullableByKey(session, project.key())).isNull(); assertThat(tester.get(ComponentDao.class).getNullableByKey(session, file.key())).isNull(); - assertThat(tester.get(IssueDao.class).getNullableByKey(session, file.key())).isNull(); + assertThat(tester.get(IssueDao.class).selectNullableByKey(session, file.key())).isNull(); assertThat(tester.get(RuleDao.class).getNullableByKey(session, rule.getKey())).isNull(); // Nothing should be removed from indexes @@ -118,7 +120,7 @@ public class BackendCleanupMediumTest { // Nothing should be removed from db assertThat(tester.get(ComponentDao.class).getNullableByKey(session, project.key())).isNotNull(); assertThat(tester.get(ComponentDao.class).getNullableByKey(session, file.key())).isNotNull(); - assertThat(tester.get(IssueDao.class).getNullableByKey(session, issue.getKey())).isNotNull(); + assertThat(tester.get(IssueDao.class).selectNullableByKey(session, issue.getKey())).isNotNull(); assertThat(tester.get(RuleDao.class).getNullableByKey(session, rule.getKey())).isNotNull(); // Everything should be removed from indexes @@ -134,7 +136,7 @@ public class BackendCleanupMediumTest { // Everything should be removed from db assertThat(tester.get(ComponentDao.class).getNullableByKey(session, project.key())).isNull(); assertThat(tester.get(ComponentDao.class).getNullableByKey(session, file.key())).isNull(); - assertThat(tester.get(IssueDao.class).getNullableByKey(session, file.key())).isNull(); + assertThat(tester.get(IssueDao.class).selectNullableByKey(session, file.key())).isNull(); assertThat(tester.get(RuleDao.class).getNullableByKey(session, rule.getKey())).isNull(); // Everything should be removed from indexes @@ -150,7 +152,7 @@ public class BackendCleanupMediumTest { // Every projects and issues are removed (from db and indexes) assertThat(tester.get(ComponentDao.class).getNullableByKey(session, project.key())).isNull(); assertThat(tester.get(ComponentDao.class).getNullableByKey(session, file.key())).isNull(); - assertThat(tester.get(IssueDao.class).getNullableByKey(session, issue.getKey())).isNull(); + assertThat(tester.get(IssueDao.class).selectNullableByKey(session, issue.getKey())).isNull(); assertThat(tester.get(IssueIndex.class).getNullableByKey(issue.getKey())).isNull(); // Every rules should not be removed (from db and indexes) diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest/before.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest/before.xml new file mode 100644 index 00000000000..2c029ebf895 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest/before.xml @@ -0,0 +1,51 @@ + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest/schema.sql new file mode 100644 index 00000000000..66c7d3a7a9f --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedIssueLongDatesTest/schema.sql @@ -0,0 +1,28 @@ +CREATE TABLE "ISSUES" ( + "ID" BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "KEE" VARCHAR(50) UNIQUE NOT NULL, + "COMPONENT_ID" INTEGER NOT NULL, + "ROOT_COMPONENT_ID" INTEGER, + "RULE_ID" INTEGER, + "SEVERITY" VARCHAR(10), + "MANUAL_SEVERITY" BOOLEAN NOT NULL, + "MESSAGE" VARCHAR(4000), + "LINE" INTEGER, + "EFFORT_TO_FIX" DOUBLE, + "TECHNICAL_DEBT" INTEGER, + "STATUS" VARCHAR(20), + "RESOLUTION" VARCHAR(20), + "CHECKSUM" VARCHAR(1000), + "REPORTER" VARCHAR(255), + "ASSIGNEE" VARCHAR(255), + "AUTHOR_LOGIN" VARCHAR(255), + "ACTION_PLAN_KEY" VARCHAR(50) NULL, + "ISSUE_ATTRIBUTES" VARCHAR(4000), + "ISSUE_CREATION_DATE" TIMESTAMP, + "ISSUE_CLOSE_DATE" TIMESTAMP, + "ISSUE_UPDATE_DATE" TIMESTAMP, + "CREATED_AT" TIMESTAMP, + "UPDATED_AT" TIMESTAMP, + "CREATED_AT_MS" BIGINT, + "UPDATED_AT_MS" BIGINT +); diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_insert_new_issues-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_insert_new_issues-result.xml index 59d288be419..bae0752883b 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_insert_new_issues-result.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_insert_new_issues-result.xml @@ -10,8 +10,8 @@ component_id="100" root_component_id="10" rule_id="200" - created_at="[null]" - updated_at="[null]" + created_at="1000000000" + updated_at="1000000000" reporter="emmerik" issue_attributes="foo=bar" action_plan_key="[null]" diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues-result.xml index 82db8eecb31..21a6c8061e3 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues-result.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues-result.xml @@ -15,8 +15,8 @@ component_id="100" root_component_id="10" rule_id="200" - created_at="2013-05-18" - updated_at="2013-05-18" + created_at="1000000000" + updated_at="2000000000" reporter="emmerik" issue_attributes="foo=bar" action_plan_key="[null]" diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues.xml index 15d6a321478..a655b018921 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/ServerIssueStorageTest/should_update_issues.xml @@ -22,8 +22,8 @@ component_id="100" root_component_id="10" rule_id="200" - created_at="2010-01-01" - updated_at="2011-02-02" + created_at="1000000000" + updated_at="1000000000" reporter="emmerik" issue_attributes="foo=bar" action_plan_key="[null]" diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/find_after_dates_with_project.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/find_after_dates_with_project.xml deleted file mode 100644 index d58c6186d19..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/find_after_dates_with_project.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/find_by_action_plan.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/find_by_action_plan.xml index 6ba7716170a..b9421a4cb4c 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/find_by_action_plan.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/find_by_action_plan.xml @@ -22,8 +22,8 @@ issue_creation_date="2013-04-16" issue_update_date="2013-04-16" issue_close_date="2013-04-16" - created_at="2013-04-16" - updated_at="2013-04-16" + created_at="1400000000000" + updated_at="1400000000000" /> diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/get_by_key.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/get_by_key.xml index 9a5cc947917..21284a7aa0b 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/get_by_key.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/get_by_key.xml @@ -21,8 +21,8 @@ issue_creation_date="2013-04-16" issue_update_date="2013-04-16" issue_close_date="2013-04-16" - created_at="2013-04-16" - updated_at="2013-04-16" + created_at="1400000000000" + updated_at="1450000000000" /> diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/insert-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/insert-result.xml index 3d736381a9e..6e6ab5c5cfb 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/insert-result.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/insert-result.xml @@ -21,8 +21,8 @@ issue_creation_date="2013-05-18" issue_update_date="2013-05-19" issue_close_date="2013-05-20" - created_at="2013-05-21" - updated_at="2013-05-22" + created_at="1400000000000" + updated_at="1450000000000" action_plan_key="current_sprint" /> diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/some_issues.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/some_issues.xml deleted file mode 100644 index 96ada68ee82..00000000000 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/some_issues.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - - - - - - diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update-result.xml index a923dfa54e0..ea43e6e66ea 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update-result.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update-result.xml @@ -21,8 +21,8 @@ issue_creation_date="2013-05-18" issue_update_date="2013-05-19" issue_close_date="2013-05-20" - created_at="[null]" - updated_at="2013-05-22" + created_at="1400000000000" + updated_at="1450000000000" action_plan_key="current_sprint" /> diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update.xml index 4c6f701dc2c..e4bf4848927 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/db/IssueDaoTest/update.xml @@ -21,8 +21,8 @@ issue_creation_date="[null]" issue_update_date="[null]" issue_close_date="[null]" - created_at="[null]" - updated_at="2009-01-01" + created_at="1400000000000" + updated_at="1400000000000" action_plan_key="[null]" /> diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueIndexerTest/index.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueIndexerTest/index.xml index 3dc5817d425..f04a8fd87fe 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueIndexerTest/index.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueIndexerTest/index.xml @@ -26,8 +26,8 @@ reporter="[null]" issue_attributes="JIRA=http://jira.com" action_plan_key="[null]" - created_at="2005-05-12" - updated_at="2013-05-18" + created_at="1000000000" + updated_at="2000000000" issue_creation_date="2005-05-12 00:00:00.0" issue_update_date="2013-05-18 00:00:00.0" issue_close_date="[null]" diff --git a/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueResultSetIteratorTest/shared.xml b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueResultSetIteratorTest/shared.xml index 8bf47afac4f..3341306e09d 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueResultSetIteratorTest/shared.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/issue/index/IssueResultSetIteratorTest/shared.xml @@ -25,8 +25,8 @@ reporter="[null]" issue_attributes="JIRA=http://jira.com" action_plan_key="PLAN1" - created_at="2005-05-12" - updated_at="2013-01-01" + created_at="1400000000000" + updated_at="1400000000000" issue_creation_date="2005-05-12 00:00:00.0" issue_update_date="2013-01-01 00:00:00.0" issue_close_date="[null]"/> @@ -51,8 +51,8 @@ reporter="[null]" issue_attributes="JIRA=http://jira.com" action_plan_key="PLAN2" - created_at="2005-05-12" - updated_at="2014-05-18" + created_at="1400000000000" + updated_at="1450000000000" issue_creation_date="2005-05-12 00:00:00.0" issue_update_date="2013-05-18 00:00:00.0" issue_close_date="[null]"/> diff --git a/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/718_add_issue_long_dates.rb b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/718_add_issue_long_dates.rb new file mode 100644 index 00000000000..bf0288c7f12 --- /dev/null +++ b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/718_add_issue_long_dates.rb @@ -0,0 +1,28 @@ +# +# SonarQube, open source software quality management tool. +# Copyright (C) 2008-2014 SonarSource +# mailto:contact AT sonarsource DOT com +# +# SonarQube is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# SonarQube is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +class AddIssueLongDates < ActiveRecord::Migration + + def self.up + add_column 'issues', :created_at_ms, :big_integer, :null => true + add_column 'issues', :updated_at_ms, :big_integer, :null => true + end +end + diff --git a/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/719_feed_issue_long_dates.rb b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/719_feed_issue_long_dates.rb new file mode 100644 index 00000000000..fbabcaa7633 --- /dev/null +++ b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/719_feed_issue_long_dates.rb @@ -0,0 +1,31 @@ +# +# SonarQube, open source software quality management tool. +# Copyright (C) 2008-2014 SonarSource +# mailto:contact AT sonarsource DOT com +# +# SonarQube is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# SonarQube is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# +# SonarQube 5.0 +# +class FeedIssueLongDates < ActiveRecord::Migration + + def self.up + execute_java_migration('org.sonar.server.db.migrations.v50.FeedIssueLongDates') + end + +end + diff --git a/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/720_rename_issue_long_dates.rb b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/720_rename_issue_long_dates.rb new file mode 100644 index 00000000000..e81c5659a0b --- /dev/null +++ b/server/sonar-web/src/main/webapp/WEB-INF/db/migrate/720_rename_issue_long_dates.rb @@ -0,0 +1,32 @@ +# +# SonarQube, open source software quality management tool. +# Copyright (C) 2008-2014 SonarSource +# mailto:contact AT sonarsource DOT com +# +# SonarQube is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# SonarQube is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +class RenameIssueLongDates < ActiveRecord::Migration + + def self.up + remove_index 'issues', :name => 'issues_updated_at' + remove_column 'issues', 'created_at' + remove_column 'issues', 'updated_at' + rename_column 'issues', 'created_at_ms', 'created_at' + rename_column 'issues', 'updated_at_ms', 'updated_at' + add_index 'issues', 'updated_at', :name => 'issues_updated_at' + end +end + diff --git a/sonar-batch/src/main/java/org/sonar/batch/issue/ScanIssueStorage.java b/sonar-batch/src/main/java/org/sonar/batch/issue/ScanIssueStorage.java index 1247db7f69e..fd8d97ec68c 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/issue/ScanIssueStorage.java +++ b/sonar-batch/src/main/java/org/sonar/batch/issue/ScanIssueStorage.java @@ -37,8 +37,6 @@ import org.sonar.core.resource.ResourceDao; import org.sonar.core.resource.ResourceDto; import org.sonar.core.resource.ResourceQuery; -import java.util.Date; - public class ScanIssueStorage extends IssueStorage implements BatchComponent { private final SnapshotCache snapshotCache; @@ -54,7 +52,7 @@ public class ScanIssueStorage extends IssueStorage implements BatchComponent { } @Override - protected void doInsert(DbSession session, Date now, DefaultIssue issue) { + protected void doInsert(DbSession session, long now, DefaultIssue issue) { IssueMapper issueMapper = session.getMapper(IssueMapper.class); long componentId = componentId(issue); long projectId = projectId(); @@ -64,7 +62,7 @@ public class ScanIssueStorage extends IssueStorage implements BatchComponent { } @Override - protected void doUpdate(DbSession session, Date now, DefaultIssue issue) { + protected void doUpdate(DbSession session, long now, DefaultIssue issue) { IssueMapper issueMapper = session.getMapper(IssueMapper.class); IssueDto dto = IssueDto.toDtoForUpdate(issue, projectId(), now); if (Issue.STATUS_CLOSED.equals(issue.status()) || issue.selectedAt() == null) { diff --git a/sonar-batch/src/test/java/org/sonar/batch/issue/ScanIssueStorageTest.java b/sonar-batch/src/test/java/org/sonar/batch/issue/ScanIssueStorageTest.java index 7c7972f75f2..386c4949251 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/issue/ScanIssueStorageTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/issue/ScanIssueStorageTest.java @@ -206,8 +206,8 @@ public class ScanIssueStorageTest extends AbstractDaoTestCase { .setRuleKey(RuleKey.of("squid", "AvoidCycles")) .setComponentKey("struts:Action") - // issue in database has been updated in 2013, after the loading by scan - .setSelectedAt(DateUtils.parseDate("2005-01-01")) + // issue in database has been updated in 2015, after the loading by scan + .setSelectedAt(1400000000000L) // fields to be updated .setLine(444) diff --git a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates-result.xml index ce45d9f5bf4..c18f8076427 100644 --- a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates-result.xml +++ b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates-result.xml @@ -24,8 +24,8 @@ reporter="[null]" issue_attributes="JIRA=http://jira.com" action_plan_key="[null]" - created_at="2005-05-12" - updated_at="2013-05-18" + created_at="1400000000000" + updated_at="1400000000000" issue_creation_date="2005-05-12 00:00:00.0" issue_update_date="2013-05-18 00:00:00.0" issue_close_date="[null]" diff --git a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates.xml b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates.xml index 0e41f5007cd..0ddfc4586ab 100644 --- a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates.xml +++ b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_resolve_conflicts_on_updates.xml @@ -25,8 +25,8 @@ reporter="[null]" issue_attributes="" action_plan_key="[null]" - created_at="2005-05-12" - updated_at="2013-05-18" + created_at="1400000000000" + updated_at="1500000000000" issue_creation_date="2005-05-12 00:00:00.0" issue_update_date="2013-05-18 00:00:00.0" issue_close_date="[null]" diff --git a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues-result.xml index 82db8eecb31..7cae675e982 100644 --- a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues-result.xml +++ b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues-result.xml @@ -15,8 +15,8 @@ component_id="100" root_component_id="10" rule_id="200" - created_at="2013-05-18" - updated_at="2013-05-18" + created_at="1400000000000" + updated_at="1400000000000" reporter="emmerik" issue_attributes="foo=bar" action_plan_key="[null]" diff --git a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues.xml b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues.xml index 3e603048abb..5fd4a9e813c 100644 --- a/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues.xml +++ b/sonar-batch/src/test/resources/org/sonar/batch/issue/ScanIssueStorageTest/should_update_issues.xml @@ -19,8 +19,8 @@ component_id="100" root_component_id="10" rule_id="200" - created_at="2010-01-01" - updated_at="2011-02-02" + created_at="1400000000000" + updated_at="1400000000000" reporter="emmerik" issue_attributes="foo=bar" action_plan_key="[null]" diff --git a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDao.java b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDao.java index a8cd6443445..7363ea8426f 100644 --- a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDao.java +++ b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDao.java @@ -47,10 +47,9 @@ public class IssueDao implements BatchComponent, ServerComponent { @CheckForNull public IssueDto selectByKey(String key) { - SqlSession session = mybatis.openSession(false); + DbSession session = mybatis.openSession(false); try { - IssueMapper mapper = session.getMapper(IssueMapper.class); - return mapper.selectByKey(key); + return mapper(session).selectByKey(key); } finally { MyBatis.closeQuietly(session); } @@ -68,11 +67,16 @@ public class IssueDao implements BatchComponent, ServerComponent { // TODO replace by aggregation in IssueIndex public List findRulesByComponent(String componentKey, @Nullable Date createdAtOrAfter, DbSession session) { - return session.getMapper(IssueMapper.class).findRulesByComponent(componentKey, createdAtOrAfter); + return mapper(session).findRulesByComponent(componentKey, createdAtOrAfter); } // TODO replace by aggregation in IssueIndex public List findSeveritiesByComponent(String componentKey, @Nullable Date createdAtOrAfter, DbSession session) { - return session.getMapper(IssueMapper.class).findSeveritiesByComponent(componentKey, createdAtOrAfter); + return mapper(session).findSeveritiesByComponent(componentKey, createdAtOrAfter); + } + + protected IssueMapper mapper(DbSession session) { + return session.getMapper(IssueMapper.class); } + } diff --git a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java index 7cff2f63ff4..b201abbf812 100644 --- a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java +++ b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java @@ -30,7 +30,6 @@ import org.sonar.api.utils.Duration; import org.sonar.api.utils.KeyValueFormat; import org.sonar.api.utils.internal.Uuids; import org.sonar.core.component.ComponentDto; -import org.sonar.core.persistence.Dto; import org.sonar.core.rule.RuleDto; import javax.annotation.CheckForNull; @@ -42,7 +41,7 @@ import java.util.Date; /** * @since 3.6 */ -public final class IssueDto extends Dto implements Serializable { +public final class IssueDto implements Serializable { private Long id; private String kee; @@ -63,6 +62,8 @@ public final class IssueDto extends Dto implements Serializable { private String authorLogin; private String actionPlanKey; private String issueAttributes; + private long createdAt; + private long updatedAt; // functional dates private Date issueCreationDate; @@ -72,7 +73,7 @@ public final class IssueDto extends Dto implements Serializable { /** * Temporary date used only during scan */ - private Date selectedAt; + private Long selectedAt; // joins private String ruleKey; @@ -86,7 +87,6 @@ public final class IssueDto extends Dto implements Serializable { private String projectUuid; private String filePath; - @Override public String getKey() { return getKee(); } @@ -308,15 +308,27 @@ public final class IssueDto extends Dto implements Serializable { return this; } - @Override - public IssueDto setCreatedAt(Date createdAt) { - super.setCreatedAt(createdAt); + public long getCreatedAt() { + return createdAt; + } + + /** + * Technical date + */ + public IssueDto setCreatedAt(long createdAt) { + this.createdAt = createdAt; return this; } - @Override - public IssueDto setUpdatedAt(@Nullable Date updatedAt) { - super.setUpdatedAt(updatedAt); + public long getUpdatedAt() { + return updatedAt; + } + + /** + * Technical date + */ + public IssueDto setUpdatedAt(long updatedAt) { + this.updatedAt = updatedAt; return this; } @@ -355,11 +367,11 @@ public final class IssueDto extends Dto implements Serializable { return ruleRepo; } - public RuleKey getRuleKey(){ + public RuleKey getRuleKey() { return RuleKey.of(ruleRepo, ruleKey); } - public String getLanguage(){ + public String getLanguage() { return language; } @@ -401,11 +413,11 @@ public final class IssueDto extends Dto implements Serializable { } @CheckForNull - public Date getSelectedAt() { + public Long getSelectedAt() { return selectedAt; } - public IssueDto setSelectedAt(@Nullable Date d) { + public IssueDto setSelectedAt(@Nullable Long d) { this.selectedAt = d; return this; } @@ -518,7 +530,7 @@ public final class IssueDto extends Dto implements Serializable { /** * On batch side, component keys and uuid are useless */ - public static IssueDto toDtoForBatchInsert(DefaultIssue issue, Long componentId, Long rootComponentId, Integer ruleId, Date now) { + public static IssueDto toDtoForBatchInsert(DefaultIssue issue, Long componentId, Long rootComponentId, Integer ruleId, long now) { return new IssueDto() .setKee(issue.key()) .setLine(issue.line()) @@ -550,6 +562,8 @@ public final class IssueDto extends Dto implements Serializable { .setIssueCloseDate(issue.closeDate()) .setIssueUpdateDate(issue.updateDate()) .setSelectedAt(issue.selectedAt()) + + // technical dates .setCreatedAt(now) .setUpdatedAt(now); } @@ -557,13 +571,13 @@ public final class IssueDto extends Dto implements Serializable { /** * On server side, we need component keys and uuid */ - public static IssueDto toDtoForServerInsert(DefaultIssue issue, ComponentDto component, ComponentDto project, Integer ruleId, Date now) { + public static IssueDto toDtoForServerInsert(DefaultIssue issue, ComponentDto component, ComponentDto project, Integer ruleId, long now) { return toDtoForBatchInsert(issue, component.getId(), project.getId(), ruleId, now) .setComponent(component) .setProject(project); } - public static IssueDto toDtoForUpdate(DefaultIssue issue, Long rootComponentId, Date now) { + public static IssueDto toDtoForUpdate(DefaultIssue issue, Long rootComponentId, long now) { // Invariant fields, like key and rule, can't be updated return new IssueDto() .setKee(issue.key()) @@ -593,6 +607,8 @@ public final class IssueDto extends Dto implements Serializable { .setIssueCloseDate(issue.closeDate()) .setIssueUpdateDate(issue.updateDate()) .setSelectedAt(issue.selectedAt()) + + // technical date .setUpdatedAt(now); } diff --git a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueMapper.java b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueMapper.java index 5e51b8c3a56..ac41ee8639e 100644 --- a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueMapper.java @@ -23,8 +23,6 @@ import org.apache.ibatis.annotations.Param; import org.sonar.core.rule.RuleDto; import javax.annotation.Nullable; - -import java.sql.Timestamp; import java.util.Collection; import java.util.Date; import java.util.List; @@ -47,5 +45,4 @@ public interface IssueMapper { int updateIfBeforeSelectedDate(IssueDto issue); - List selectAfterDate(@Nullable @Param("date") Timestamp timestamp, @Nullable @Param("project") String projectUuid); } diff --git a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueStorage.java b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueStorage.java index c89c87b417d..7455bd4bd07 100644 --- a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueStorage.java +++ b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueStorage.java @@ -30,7 +30,6 @@ import org.sonar.core.persistence.BatchSession; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.MyBatis; -import java.util.Date; import java.util.List; import static com.google.common.collect.Lists.newArrayList; @@ -59,28 +58,32 @@ public abstract class IssueStorage { } public void save(DbSession session, DefaultIssue issue) { - save(session, newArrayList(issue)); + doSave(session, newArrayList(issue)); } public void save(Iterable issues) { DbSession session = mybatis.openSession(true); try { - save(session, issues); - session.commit(); + doSave(session, issues); } finally { MyBatis.closeQuietly(session); } } - private void save(DbSession session, Iterable issues) { + private void doSave(DbSession session, Iterable issues) { // Batch session can not be used for updates. It does not return the number of updated rows, // required for detecting conflicts. - Date now = new Date(); + long now = System.currentTimeMillis(); List toBeUpdated = batchInsert(session, issues, now); update(toBeUpdated, now); + doAfterSave(); } - private List batchInsert(DbSession session, Iterable issues, Date now) { + protected void doAfterSave() { + // overridden on server-side to index ES + } + + private List batchInsert(DbSession session, Iterable issues, long now) { List toBeUpdated = newArrayList(); int count = 0; IssueChangeMapper issueChangeMapper = session.getMapper(IssueChangeMapper.class); @@ -96,12 +99,13 @@ public abstract class IssueStorage { toBeUpdated.add(issue); } } + session.commit(); return toBeUpdated; } - protected abstract void doInsert(DbSession batchSession, Date now, DefaultIssue issue); + protected abstract void doInsert(DbSession batchSession, long now, DefaultIssue issue); - private void update(List toBeUpdated, Date now) { + private void update(List toBeUpdated, long now) { if (!toBeUpdated.isEmpty()) { DbSession session = mybatis.openSession(false); try { @@ -117,7 +121,7 @@ public abstract class IssueStorage { } } - protected abstract void doUpdate(DbSession batchSession, Date now, DefaultIssue issue); + protected abstract void doUpdate(DbSession batchSession, long now, DefaultIssue issue); private void insertChanges(IssueChangeMapper mapper, DefaultIssue issue) { for (IssueComment comment : issue.comments()) { diff --git a/sonar-core/src/main/java/org/sonar/core/issue/db/UpdateConflictResolver.java b/sonar-core/src/main/java/org/sonar/core/issue/db/UpdateConflictResolver.java index 622de121842..a5fa56cfeb5 100644 --- a/sonar-core/src/main/java/org/sonar/core/issue/db/UpdateConflictResolver.java +++ b/sonar-core/src/main/java/org/sonar/core/issue/db/UpdateConflictResolver.java @@ -24,8 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.issue.internal.DefaultIssue; -import java.util.Date; - /** * See https://jira.codehaus.org/browse/SONAR-4309 * @@ -41,7 +39,7 @@ public class UpdateConflictResolver { IssueDto dbIssue = mapper.selectByKey(issue.key()); if (dbIssue != null) { mergeFields(dbIssue, issue); - mapper.update(IssueDto.toDtoForUpdate(issue, dbIssue.getProjectId(), new Date())); + mapper.update(IssueDto.toDtoForUpdate(issue, dbIssue.getProjectId(), System.currentTimeMillis())); } } diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java index 783c7433141..a0677316258 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/DatabaseVersion.java @@ -33,7 +33,8 @@ import java.util.List; */ public class DatabaseVersion implements BatchComponent, ServerComponent { - public static final int LAST_VERSION = 717; + public static final int LAST_VERSION = 720; + /** * List of all the tables. * This list is hardcoded because we didn't succeed in using java.sql.DatabaseMetaData#getTables() in the same way diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java index 71dc5863e15..2e9b914d605 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeDao.java @@ -195,7 +195,7 @@ public class PurgeDao { mapper.setSnapshotIsLastToFalse(resourceId); mapper.deleteFileSourcesByUuid(resourceIdUuid.getUuid()); mapper.disableResource(resourceId); - mapper.resolveResourceIssuesNotAlreadyResolved(resourceId, new Date(system2.now())); + mapper.resolveResourceIssuesNotAlreadyResolved(resourceId, new Date(system2.now()), system2.now()); } public PurgeDao deleteSnapshots(PurgeSnapshotQuery query) { diff --git a/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java b/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java index 4ece70e8e4a..81e0d2d973e 100644 --- a/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java +++ b/sonar-core/src/main/java/org/sonar/core/purge/PurgeMapper.java @@ -60,7 +60,7 @@ public interface PurgeMapper { void disableResource(long resourceId); - void resolveResourceIssuesNotAlreadyResolved(@Param("resourceId") long resourceId, @Param("date") Date date); + void resolveResourceIssuesNotAlreadyResolved(@Param("resourceId") long resourceId, @Param("date") Date date, @Param("dateAsLong") Long dateAsLong); void deleteResourceIndex(@Param("resourceIds") List resourceIds); diff --git a/sonar-core/src/main/resources/org/sonar/core/issue/db/IssueMapper.xml b/sonar-core/src/main/resources/org/sonar/core/issue/db/IssueMapper.xml index b781a3ff962..26f08500466 100644 --- a/sonar-core/src/main/resources/org/sonar/core/issue/db/IssueMapper.xml +++ b/sonar-core/src/main/resources/org/sonar/core/issue/db/IssueMapper.xml @@ -141,23 +141,6 @@ where i.kee=#{kee} - -