import org.sonar.api.utils.System2;
import org.sonar.core.component.ComponentDto;
import org.sonar.core.component.FilePathWithHashDto;
+import org.sonar.core.component.UuidWithProjectUuidDto;
import org.sonar.core.component.db.ComponentMapper;
import org.sonar.core.persistence.DaoComponent;
import org.sonar.core.persistence.DbSession;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import static com.google.common.collect.Lists.newArrayList;
return mapper(session).findProjectUuids();
}
- public List<Map<String, String>> selectAllViewsAndSubViews(DbSession session) {
+ public List<UuidWithProjectUuidDto> selectAllViewsAndSubViews(DbSession session) {
return mapper(session).selectAllViewsAndSubViews(Qualifiers.VIEW, Qualifiers.SUBVIEW);
}
import org.sonar.core.issue.db.UpdateConflictResolver;
import org.sonar.server.computation.issue.*;
import org.sonar.server.computation.step.ComputationSteps;
+import org.sonar.server.view.index.ViewIndex;
import java.util.Arrays;
import java.util.List;
RuleCache.class,
RuleCacheLoader.class,
IssueCache.class,
- UpdateConflictResolver.class);
+ UpdateConflictResolver.class,
+
+ // views
+ ViewIndex.class);
}
}
IndexSourceLinesStep.class,
IndexViewsStep.class,
+ // Purge of removed views has to be done after Views has been indexed
+ PurgeRemovedViewsStep.class,
+
// notifications are sent at the end, so that webapp displays up-to-date information
SendIssueNotificationsStep.class);
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.computation.step;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.sonar.api.resources.Qualifiers;
+import org.sonar.core.component.UuidWithProjectUuidDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.server.computation.ComputationContext;
+import org.sonar.server.db.DbClient;
+import org.sonar.server.view.index.ViewIndex;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+import java.util.Set;
+
+import static com.google.common.collect.Sets.newHashSet;
+
+/**
+ * This step will remove every Views and Sub Views from the index that do not exists in the db.
+ * As it's executed on each analysis, it means that when the Views task is executed on every views, this step will be executed on each views !
+ *
+ * A more optimized approach would be to execute this step only once at this end of the Views task.
+ */
+public class PurgeRemovedViewsStep implements ComputationStep {
+
+ private final DbClient dbClient;
+ private final ViewIndex index;
+
+ public PurgeRemovedViewsStep(ViewIndex index, DbClient dbClient) {
+ this.index = index;
+ this.dbClient = dbClient;
+ }
+
+ @Override
+ public void execute(ComputationContext context) {
+ if (context.getProject().qualifier().equals(Qualifiers.VIEW)) {
+ purgeRemovedViews();
+ }
+ }
+
+ private void purgeRemovedViews() {
+ DbSession session = dbClient.openSession(false);
+ try {
+ List<UuidWithProjectUuidDto> uuidWithProjectUuidDtos = dbClient.componentDao().selectAllViewsAndSubViews(session);
+ Set<String> viewUuidsInDb = newHashSet(Iterables.transform(uuidWithProjectUuidDtos, new Function<UuidWithProjectUuidDto, String>() {
+ @Override
+ public String apply(@Nullable UuidWithProjectUuidDto input) {
+ return input != null ? input.getUuid() : null;
+ }
+ }));
+ Set<String> viewUuidsInIndex = newHashSet(index.findAllViewUuids());
+ Set<String> viewsToRemove = Sets.difference(viewUuidsInIndex, viewUuidsInDb);
+ index.delete(viewsToRemove);
+ } finally {
+ session.close();
+ }
+ }
+
+ @Override
+ public String getDescription() {
+ return "Purge removed views";
+ }
+}
package org.sonar.server.view.index;
+import com.google.common.collect.Maps;
import org.sonar.server.search.BaseDoc;
import java.util.List;
super(fields);
}
+ public ViewDoc() {
+ super(Maps.<String, Object>newHashMap());
+ }
+
public String uuid() {
return getField(ViewIndexDefinition.FIELD_UUID);
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.view.index;
+
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.sonar.api.ServerComponent;
+import org.sonar.server.es.EsClient;
+
+import java.util.Collection;
+import java.util.List;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+public class ViewIndex implements ServerComponent {
+
+ private final int SCROLL_TIME_IN_MINUTES = 3;
+
+ private final EsClient esClient;
+
+ public ViewIndex(EsClient esClient) {
+ this.esClient = esClient;
+ }
+
+ public List<String> findAllViewUuids() {
+ SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.INDEX)
+ .setTypes(ViewIndexDefinition.TYPE_VIEW)
+ .setSearchType(SearchType.SCAN)
+ .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES))
+ .setFetchSource(ViewIndexDefinition.FIELD_UUID, null)
+ .setSize(100)
+ .setQuery(QueryBuilders.matchAllQuery());
+
+ SearchResponse response = esSearch.get();
+ List<String> result = newArrayList();
+ while (true) {
+ List<SearchHit> hits = newArrayList(response.getHits());
+ for (SearchHit hit : hits) {
+ result.add((String) hit.getSource().get(ViewIndexDefinition.FIELD_UUID));
+ }
+ response = esClient.prepareSearchScroll(response.getScrollId())
+ .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES))
+ .get();
+ // Break condition: No hits are returned
+ if (response.getHits().getHits().length == 0) {
+ break;
+ }
+ }
+ return result;
+ }
+
+ public void delete(Collection<String> viewUuids) {
+ esClient
+ .prepareDeleteByQuery(ViewIndexDefinition.INDEX)
+ .setTypes(ViewIndexDefinition.TYPE_VIEW)
+ .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(),
+ FilterBuilders.termsFilter(ViewIndexDefinition.FIELD_UUID, viewUuids)
+ ))
+ .get();
+ }
+}
package org.sonar.server.view.index;
-import com.google.common.collect.Maps;
import org.elasticsearch.action.update.UpdateRequest;
import org.sonar.core.component.ComponentDto;
+import org.sonar.core.component.UuidWithProjectUuidDto;
import org.sonar.core.persistence.DbSession;
import org.sonar.server.db.DbClient;
import org.sonar.server.es.BaseIndexer;
DbSession dbSession = dbClient.openSession(false);
try {
Map<String, String> viewAndProjectViewUuidMap = newHashMap();
- for (Map<String, String> viewsMap : dbClient.componentDao().selectAllViewsAndSubViews(dbSession)) {
- viewAndProjectViewUuidMap.put(viewsMap.get("uuid"), viewsMap.get("projectUuid"));
+ for (UuidWithProjectUuidDto uuidWithProjectUuidDto : dbClient.componentDao().selectAllViewsAndSubViews(dbSession)) {
+ viewAndProjectViewUuidMap.put(uuidWithProjectUuidDto.getUuid(), uuidWithProjectUuidDto.getProjectUuid());
}
index(dbSession, viewAndProjectViewUuidMap);
} finally {
private void doIndex(DbSession dbSession, BulkIndexer bulk, String uuid, String projectUuid) {
List<String> projects = dbClient.componentDao().selectProjectsFromView(dbSession, uuid, projectUuid);
- bulk.add(newUpsertRequest(new ViewDoc(Maps.<String, Object>newHashMap())
+ bulk.add(newUpsertRequest(new ViewDoc()
.setUuid(uuid)
.setProjects(projects)));
}
}
public static ComponentDto newModuleDto(ComponentDto subProjectOrProject) {
- String uuid = Uuids.create();
+ return newModuleDto(subProjectOrProject, Uuids.create());
+ }
+
+ public static ComponentDto newModuleDto(ComponentDto subProjectOrProject, String uuid) {
return newComponent(subProjectOrProject, uuid)
.setKey("KEY_" + uuid)
.setName("NAME_" + uuid)
package org.sonar.server.component.db;
-import com.google.common.collect.ImmutableMap;
import org.apache.ibatis.exceptions.PersistenceException;
import org.junit.After;
import org.junit.Before;
public void select_views_and_sub_views() {
setupData("shared_views");
- assertThat(dao.selectAllViewsAndSubViews(session)).contains(
- ImmutableMap.of("uuid", "ABCD", "projectUuid", "ABCD"),
- ImmutableMap.of("uuid", "EFGH", "projectUuid", "EFGH"),
- ImmutableMap.of("uuid", "FGHI", "projectUuid", "EFGH"),
- ImmutableMap.of("uuid", "IJKL", "projectUuid", "IJKL")
- );
+ assertThat(dao.selectAllViewsAndSubViews(session)).extracting("uuid").containsOnly("ABCD", "EFGH", "FGHI", "IJKL");
+ assertThat(dao.selectAllViewsAndSubViews(session)).extracting("projectUuid").containsOnly("ABCD", "EFGH", "IJKL");
}
@Test
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.computation.step;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import org.elasticsearch.search.SearchHit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.config.Settings;
+import org.sonar.api.resources.Qualifiers;
+import org.sonar.core.component.ComponentDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.core.persistence.DbTester;
+import org.sonar.server.component.ComponentTesting;
+import org.sonar.server.component.db.ComponentDao;
+import org.sonar.server.computation.ComputationContext;
+import org.sonar.server.db.DbClient;
+import org.sonar.server.es.EsTester;
+import org.sonar.server.issue.db.IssueDao;
+import org.sonar.server.view.index.ViewDoc;
+import org.sonar.server.view.index.ViewIndex;
+import org.sonar.server.view.index.ViewIndexDefinition;
+
+import java.util.List;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class PurgeRemovedViewsStepTest {
+
+ @Rule
+ public EsTester esTester = new EsTester().addDefinitions(new ViewIndexDefinition(new Settings()));
+
+ @Rule
+ public DbTester db = new DbTester();
+
+ ComputationContext context;
+
+ DbSession session;
+
+ DbClient dbClient;
+
+ PurgeRemovedViewsStep step;
+
+ @Before
+ public void setUp() {
+ context = mock(ComputationContext.class);
+ session = db.myBatis().openSession(false);
+ dbClient = new DbClient(db.database(), db.myBatis(), new IssueDao(db.myBatis()), new ComponentDao());
+ step = new PurgeRemovedViewsStep(new ViewIndex(esTester.client()), dbClient);
+ }
+
+ @After
+ public void after() {
+ this.session.close();
+ }
+
+ @Test
+ public void purge_removed_views() throws Exception {
+ when(context.getProject()).thenReturn(ComponentTesting.newProjectDto("DBCA").setQualifier(Qualifiers.VIEW));
+
+ esTester.putDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW,
+ new ViewDoc().setUuid("ABCD").getFields(),
+ new ViewDoc().setUuid("BCDE").getFields(),
+ // Should be removed as it no more exists in db
+ new ViewDoc().setUuid("CDEF").getFields());
+
+ ComponentDto view = ComponentTesting.newProjectDto("ABCD").setQualifier(Qualifiers.VIEW);
+ ComponentDto subView = ComponentTesting.newModuleDto(view, "BCDE").setQualifier(Qualifiers.SUBVIEW);
+ dbClient.componentDao().insert(session, view, subView);
+ session.commit();
+
+ step.execute(context);
+
+ List<SearchHit> results = esTester.getDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW);
+ List<String> viewUuids = newArrayList(Iterables.transform(results, new Function<SearchHit, String>() {
+ @Override
+ public String apply(SearchHit input) {
+ return new ViewDoc(input.getSource()).uuid();
+ }
+ }));
+ assertThat(viewUuids).containsOnly("ABCD", "BCDE");
+ }
+
+ @Test
+ public void nothing_to_do_when_not_analysing_view() throws Exception {
+ when(context.getProject()).thenReturn(ComponentTesting.newProjectDto("DBCA"));
+
+ esTester.putDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW,
+ new ViewDoc().setUuid("ABCD").getFields(),
+ // This vies does not exists in db...
+ new ViewDoc().setUuid("BCDE").getFields());
+
+ dbClient.componentDao().insert(session, ComponentTesting.newProjectDto("ABCD").setQualifier(Qualifiers.VIEW));
+ session.commit();
+
+ step.execute(context);
+
+ // ... But it should not be removed as the project of the context is not a view
+ assertThat(esTester.countDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW)).isEqualTo(2);
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.view.index;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.config.Settings;
+import org.sonar.server.es.EsTester;
+
+import java.util.List;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ViewIndexTest {
+
+ @Rule
+ public EsTester esTester = new EsTester().addDefinitions(new ViewIndexDefinition(new Settings()));
+
+ private ViewIndex index;
+
+ @Before
+ public void setUp() {
+ index = new ViewIndex(esTester.client());
+ }
+
+ @Test
+ public void find_all_view_uuids() throws Exception {
+ esTester.putDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW, this.getClass(), "view1.json", "view2.json");
+
+ List<String> result = newArrayList(index.findAllViewUuids());
+
+ assertThat(result).containsOnly("fed0a543-9d9c-4af5-a4ec-450a8fe78ce7", "8d0bc2a5-bfba-464b-92de-bb170e9d978e");
+ }
+
+ @Test
+ public void not_find_all_view_uuids() throws Exception {
+ List<String> result = newArrayList(index.findAllViewUuids());
+
+ assertThat(result).isEmpty();
+ }
+
+ @Test
+ public void delete_views() throws Exception {
+ esTester.putDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW, this.getClass(), "view1.json", "view2.json");
+
+ index.delete(newArrayList("fed0a543-9d9c-4af5-a4ec-450a8fe78ce7", "8d0bc2a5-bfba-464b-92de-bb170e9d978e"));
+
+ assertThat(esTester.countDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW)).isEqualTo(0L);
+ }
+
+}
package org.sonar.server.view.index;
import com.google.common.base.Function;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.junit.Before;
import org.junit.Rule;
// Some views are not in the db
dbTester.prepareDbUnit(getClass(), "index.xml");
esTester.putDocuments(ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW,
- ImmutableMap.<String, Object>of(
- ViewIndexDefinition.FIELD_UUID, "ABCD",
- ViewIndexDefinition.FIELD_PROJECTS, newArrayList("BCDE")
- ));
+ new ViewDoc().setUuid("ABCD").setProjects(newArrayList("BCDE")).getFields());
indexer.index();
--- /dev/null
+{
+ "uuid": "fed0a543-9d9c-4af5-a4ec-450a8fe78ce7",
+ "projects": [
+ "548415bc-6626-45b1-a99a-ca77aedec45f"
+ ]
+}
--- /dev/null
+{
+ "uuid": "8d0bc2a5-bfba-464b-92de-bb170e9d978e",
+ "projects": [
+ "e5dccc4f-431a-46ba-ab55-47318c332af7",
+ "6432a311-4d1f-41fd-b90a-826130f6f890",
+ "bf93ca9b-18f6-4f5d-848b-722ca7bd67d1"
+ ]
+}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.core.component;
+
+public class UuidWithProjectUuidDto {
+
+ private String uuid;
+ private String projectUuid;
+
+ public String getProjectUuid() {
+ return projectUuid;
+ }
+
+ public UuidWithProjectUuidDto setProjectUuid(String projectUuid) {
+ this.projectUuid = projectUuid;
+ return this;
+ }
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public UuidWithProjectUuidDto setUuid(String uuid) {
+ this.uuid = uuid;
+ return this;
+ }
+}
import org.apache.ibatis.annotations.Param;
import org.sonar.core.component.ComponentDto;
import org.sonar.core.component.FilePathWithHashDto;
+import org.sonar.core.component.UuidWithProjectUuidDto;
import javax.annotation.CheckForNull;
import java.util.Collection;
import java.util.List;
-import java.util.Map;
/**
* @since 4.3
/**
* Return all views and sub views
*/
- List<Map<String, String>> selectAllViewsAndSubViews(@Param("viewQualifier") String viewQualifier, @Param("subViewQualifier") String subViewQualifier);
+ List<UuidWithProjectUuidDto> selectAllViewsAndSubViews(@Param("viewQualifier") String viewQualifier, @Param("subViewQualifier") String subViewQualifier);
/**
* Return technical projects from a view or a sub-view
import org.sonar.core.component.ComponentDto;
import org.sonar.core.component.FilePathWithHashDto;
import org.sonar.core.component.SnapshotDto;
+import org.sonar.core.component.UuidWithProjectUuidDto;
import org.sonar.core.component.db.ComponentMapper;
import org.sonar.core.component.db.SnapshotMapper;
import org.sonar.core.computation.db.AnalysisReportDto;
loadAlias(conf, "AnalysisReport", AnalysisReportDto.class);
loadAlias(conf, "IdUuidPair", IdUuidPair.class);
loadAlias(conf, "FilePathWithHash", FilePathWithHashDto.class);
+ loadAlias(conf, "UuidWithProjectUuid", UuidWithProjectUuidDto.class);
// AuthorizationMapper has to be loaded before IssueMapper because this last one used it
loadMapper(conf, "org.sonar.core.user.AuthorizationMapper");
</where>
</select>
- <select id="selectAllViewsAndSubViews" resultType="hashmap">
+ <select id="selectAllViewsAndSubViews" resultType="UuidWithProjectUuid">
SELECT v.uuid as "uuid", v.project_uuid as "projectUuid" FROM projects v
<where>
v.enabled=${_true} AND v.qualifier=#{viewQualifier} OR v.qualifier=#{subViewQualifier}
projects / sonarqube.git / commitdiff