diff options
10 files changed, 352 insertions, 13 deletions
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java index ed9e241e566..f5fa57db35b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java @@ -24,6 +24,7 @@ import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.server.issue.index.IssueAuthorizationIndexer; import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.project.es.ProjectMeasuresIndexer; import org.sonar.server.test.index.TestIndexer; import org.sonar.server.user.index.UserIndexer; import org.sonar.server.view.index.ViewIndexer; @@ -37,6 +38,7 @@ public class IndexerStartupTask { private final IssueIndexer issueIndexer; private final UserIndexer userIndexer; private final ViewIndexer viewIndexer; + private final ProjectMeasuresIndexer projectMeasuresIndexer; private final Settings settings; /** @@ -45,13 +47,14 @@ public class IndexerStartupTask { * {@link org.sonar.server.issue.index.IssueIndexer} */ public IndexerStartupTask(TestIndexer testIndexer, IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, - UserIndexer userIndexer, ViewIndexer viewIndexer, + UserIndexer userIndexer, ViewIndexer viewIndexer, ProjectMeasuresIndexer projectMeasuresIndexer, Settings settings) { this.testIndexer = testIndexer; this.issueAuthorizationIndexer = issueAuthorizationIndexer; this.issueIndexer = issueIndexer; this.userIndexer = userIndexer; this.viewIndexer = viewIndexer; + this.projectMeasuresIndexer = projectMeasuresIndexer; this.settings = settings; } @@ -70,6 +73,9 @@ public class IndexerStartupTask { LOG.info("Index views"); viewIndexer.index(); + + LOG.info("Index project measures"); + projectMeasuresIndexer.index(); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresDoc.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresDoc.java index c2e733dd1ad..e650263e76d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresDoc.java @@ -71,7 +71,7 @@ public class ProjectMeasuresDoc extends BaseDoc { @CheckForNull public Date getAnalysedAt() { - return getFieldAsDate(ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT); + return getNullableField(ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT); } public ProjectMeasuresDoc setAnalysedAt(@Nullable Date d) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java new file mode 100644 index 00000000000..2322ba94a11 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java @@ -0,0 +1,80 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.project.es; + +import java.util.Iterator; +import org.elasticsearch.action.index.IndexRequest; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.server.es.BaseIndexer; +import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.EsClient; + +import static org.sonar.server.project.es.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT; +import static org.sonar.server.project.es.ProjectMeasuresIndexDefinition.INDEX_PROJECT_MEASURES; +import static org.sonar.server.project.es.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; + +public class ProjectMeasuresIndexer extends BaseIndexer { + + private final DbClient dbClient; + + public ProjectMeasuresIndexer(DbClient dbClient, EsClient esClient) { + super(esClient, 300, INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES, FIELD_ANALYSED_AT); + this.dbClient = dbClient; + } + + @Override + protected long doIndex(long lastUpdatedAt) { + doIndex(createBulkIndexer(false)); + return 0L; + } + + private void doIndex(BulkIndexer bulk) { + DbSession dbSession = dbClient.openSession(false); + try { + ProjectMeasuresResultSetIterator rowIt = ProjectMeasuresResultSetIterator.create(dbClient, dbSession); + doIndex(bulk, rowIt); + rowIt.close(); + } finally { + dbClient.closeSession(dbSession); + } + } + + private static void doIndex(BulkIndexer bulk, Iterator<ProjectMeasuresDoc> docs) { + bulk.start(); + while (docs.hasNext()) { + ProjectMeasuresDoc doc = docs.next(); + bulk.add(newIndexRequest(doc)); + } + bulk.stop(); + } + + private BulkIndexer createBulkIndexer(boolean large) { + BulkIndexer bulk = new BulkIndexer(esClient, INDEX_PROJECT_MEASURES); + bulk.setLarge(large); + return bulk; + } + + private static IndexRequest newIndexRequest(ProjectMeasuresDoc doc) { + return new IndexRequest(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES, doc.getId()) + .source(doc.getFields()); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java new file mode 100644 index 00000000000..a936c85789a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java @@ -0,0 +1,74 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.project.es; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; +import org.apache.commons.lang.StringUtils; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Scopes; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.ResultSetIterator; + +public class ProjectMeasuresResultSetIterator extends ResultSetIterator<ProjectMeasuresDoc> { + + private static final String[] FIELDS = { + "p.uuid", + "p.kee", + "p.name", + "s.created_at" + }; + + private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM projects p " + + "LEFT OUTER JOIN snapshots s ON s.component_uuid=p.uuid AND s.islast=? " + + "WHERE p.enabled=? AND p.scope=? AND p.qualifier=?"; + + private ProjectMeasuresResultSetIterator(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + static ProjectMeasuresResultSetIterator create(DbClient dbClient, DbSession session) { + try { + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, SQL_ALL); + stmt.setBoolean(1, true); + stmt.setBoolean(2, true); + stmt.setString(3, Scopes.PROJECT); + stmt.setString(4, Qualifiers.PROJECT); + return new ProjectMeasuresResultSetIterator(stmt); + } catch (SQLException e) { + throw new IllegalStateException("Fail to prepare SQL request to select all project measures", e); + } + } + + @Override + protected ProjectMeasuresDoc read(ResultSet rs) throws SQLException { + ProjectMeasuresDoc doc = new ProjectMeasuresDoc() + .setId(rs.getString(1)) + .setKey(rs.getString(2)) + .setName(rs.getString(3)); + long analysisDate = rs.getLong(4); + doc.setAnalysedAt(analysisDate != 0 ? new Date(analysisDate) : null); + return doc; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectsEsModule.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectsEsModule.java index d1070ae1e04..fef8452c0b6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectsEsModule.java +++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectsEsModule.java @@ -27,6 +27,7 @@ public class ProjectsEsModule extends Module { protected void configureModule() { add( ProjectMeasuresIndexDefinition.class, - ProjectMeasuresIndex.class); + ProjectMeasuresIndex.class, + ProjectMeasuresIndexer.class); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresIndexerTest.java new file mode 100644 index 00000000000..8867fc6d9a5 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresIndexerTest.java @@ -0,0 +1,64 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.project.es; + +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.MapSettings; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDbTester; +import org.sonar.server.es.EsTester; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.component.ComponentTesting.newProjectDto; +import static org.sonar.server.project.es.ProjectMeasuresIndexDefinition.INDEX_PROJECT_MEASURES; +import static org.sonar.server.project.es.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; + +public class ProjectMeasuresIndexerTest { + + @Rule + public EsTester esTester = new EsTester(new ProjectMeasuresIndexDefinition(new MapSettings())); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + ComponentDbTester componentDbTester = new ComponentDbTester(dbTester); + + ProjectMeasuresIndexer underTest = new ProjectMeasuresIndexer(dbTester.getDbClient(), esTester.client()); + + @Test + public void index_nothing() { + underTest.index(); + + assertThat(esTester.countDocuments(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).isZero(); + } + + @Test + public void index_one_project() { + componentDbTester.insertProjectAndSnapshot(newProjectDto()); + + underTest.index(); + + assertThat(esTester.countDocuments(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).isEqualTo(1); + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresResultSetIteratorTest.java new file mode 100644 index 00000000000..48c889f00af --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresResultSetIteratorTest.java @@ -0,0 +1,120 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.project.es; + +import com.google.common.collect.Maps; +import java.util.Date; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.SnapshotDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.component.ComponentTesting.newDeveloper; +import static org.sonar.db.component.ComponentTesting.newProjectDto; +import static org.sonar.db.component.ComponentTesting.newView; +import static org.sonar.db.component.SnapshotTesting.newAnalysis; + +public class ProjectMeasuresResultSetIteratorTest { + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + DbClient dbClient = dbTester.getDbClient(); + DbSession dbSession = dbTester.getSession(); + + ComponentDbTester componentDbTester = new ComponentDbTester(dbTester); + + @Test + public void return_one_project_measure() { + ComponentDto project = newProjectDto().setKey("Project-Key").setName("Project Name"); + SnapshotDto analysis = componentDbTester.insertProjectAndSnapshot(project); + + Map<String, ProjectMeasuresDoc> docsById = docsById(); + + assertThat(docsById).hasSize(1); + ProjectMeasuresDoc doc = docsById.get(project.uuid()); + assertThat(doc).isNotNull(); + assertThat(doc.getId()).isEqualTo(project.uuid()); + assertThat(doc.getKey()).isEqualTo("Project-Key"); + assertThat(doc.getName()).isEqualTo("Project Name"); + assertThat(doc.getAnalysedAt()).isNotNull().isEqualTo(new Date(analysis.getCreatedAt())); + } + + @Test + public void return_many_project_measures() { + componentDbTester.insertProjectAndSnapshot(newProjectDto()); + componentDbTester.insertProjectAndSnapshot(newProjectDto()); + componentDbTester.insertProjectAndSnapshot(newProjectDto()); + + assertThat(docsById()).hasSize(3); + } + + @Test + public void return_project_without_analysis() throws Exception { + ComponentDto project = componentDbTester.insertComponent(newProjectDto()); + dbClient.snapshotDao().insert(dbSession, newAnalysis(project).setLast(false)); + dbSession.commit(); + + Map<String, ProjectMeasuresDoc> docsById = docsById(); + + assertThat(docsById).hasSize(1); + ProjectMeasuresDoc doc = docsById.get(project.uuid()); + assertThat(doc.getAnalysedAt()).isNull(); + } + + @Test + public void does_not_return_non_active_projects() throws Exception { + // Disabled project + componentDbTester.insertProjectAndSnapshot(newProjectDto().setEnabled(false)); + // Disabled project with analysis + ComponentDto project = componentDbTester.insertComponent(newProjectDto().setEnabled(false)); + dbClient.snapshotDao().insert(dbSession, newAnalysis(project)); + + // A view + componentDbTester.insertProjectAndSnapshot(newView()); + + // A developer + componentDbTester.insertProjectAndSnapshot(newDeveloper("dev")); + + dbSession.commit(); + + assertResultSetIsEmpty(); + } + + private Map<String, ProjectMeasuresDoc> docsById() { + ProjectMeasuresResultSetIterator it = ProjectMeasuresResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession()); + Map<String, ProjectMeasuresDoc> docsById = Maps.uniqueIndex(it, ProjectMeasuresDoc::getId); + it.close(); + return docsById; + } + + private void assertResultSetIsEmpty() { + assertThat(docsById()).isEmpty(); + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectsEsModuleTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectsEsModuleTest.java index 6ff53ec14c6..38f1f5e7c3e 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectsEsModuleTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectsEsModuleTest.java @@ -30,6 +30,6 @@ public class ProjectsEsModuleTest { public void verify_count_of_added_components() { ComponentContainer container = new ComponentContainer(); new ProjectsEsModule().configure(container); - assertThat(container.size()).isEqualTo(2 + 2); + assertThat(container.size()).isEqualTo(3 + 2); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java index d09dad29b98..e86e16792ce 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java @@ -19,10 +19,8 @@ */ package org.sonar.server.rule.index; -import com.google.common.base.Function; import com.google.common.collect.Maps; import java.util.Map; -import javax.annotation.Nonnull; import org.junit.Rule; import org.junit.Test; import org.sonar.api.rule.RuleKey; @@ -200,11 +198,6 @@ public class RuleResultSetIteratorTest { } private static Map<String, RuleDoc> rulesByKey(RuleResultSetIterator it) { - return Maps.uniqueIndex(it, new Function<RuleDoc, String>() { - @Override - public String apply(@Nonnull RuleDoc rule) { - return rule.key().rule(); - } - }); + return Maps.uniqueIndex(it, rule -> rule.key().rule()); } } diff --git a/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java b/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java index fd5fe9dc652..09dc1e9d0f4 100644 --- a/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java +++ b/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java @@ -328,8 +328,9 @@ public class ComponentDao implements Dao { parameters.put("qualifier", Qualifiers.PROJECT); } - public void insert(DbSession session, ComponentDto item) { + public ComponentDto insert(DbSession session, ComponentDto item) { mapper(session).insert(item); + return item; } public void insertBatch(DbSession session, ComponentDto item) { |