123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183 |
- /*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
- package org.sonar.server.view.index;
-
- import java.util.Collection;
- import java.util.HashMap;
- import java.util.List;
- import java.util.Map;
- import java.util.Set;
- import java.util.stream.Collectors;
- import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
- import org.elasticsearch.action.index.IndexRequest;
- import org.sonar.db.DbClient;
- import org.sonar.db.DbSession;
- import org.sonar.db.component.ComponentDto;
- import org.sonar.db.component.UuidWithBranchUuidDto;
- import org.sonar.db.es.EsQueueDto;
- import org.sonar.server.es.BulkIndexer;
- import org.sonar.server.es.BulkIndexer.Size;
- import org.sonar.server.es.EsClient;
- import org.sonar.server.es.IndexType;
- import org.sonar.server.es.IndexingListener;
- import org.sonar.server.es.IndexingResult;
- import org.sonar.server.es.OneToOneResilientIndexingListener;
- import org.sonar.server.es.ResilientIndexer;
-
- import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW;
-
- public class ViewIndexer implements ResilientIndexer {
-
- private final DbClient dbClient;
- private final EsClient esClient;
-
- public ViewIndexer(DbClient dbClient, EsClient esClient) {
- this.dbClient = dbClient;
- this.esClient = esClient;
- }
-
- @Override
- public Set<IndexType> getIndexTypes() {
- return Set.of(TYPE_VIEW);
- }
-
- @Override
- public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
- indexAll(Size.LARGE);
- }
-
- public void indexAll() {
- indexAll(Size.REGULAR);
- }
-
- private void indexAll(Size bulkSize) {
- try (DbSession dbSession = dbClient.openSession(false)) {
- Map<String, String> rootViewUuidByViewUuid = new HashMap<>();
- for (UuidWithBranchUuidDto uuidWithBranchUuidDto : dbClient.componentDao().selectAllViewsAndSubViews(dbSession)) {
- rootViewUuidByViewUuid.put(uuidWithBranchUuidDto.getUuid(), uuidWithBranchUuidDto.getBranchUuid());
- }
- index(dbSession, rootViewUuidByViewUuid, false, bulkSize);
- }
- }
-
- /**
- * Index a root view : it will fetch a view and its subviews from the DB and index them.
- * Used by the compute engine to reindex a root view.
- * <p/>
- * The views lookup cache will be cleared
- */
- public void index(String rootViewUuid) {
- try (DbSession dbSession = dbClient.openSession(false)) {
- Map<String, String> viewAndRootViewUuidMap = new HashMap<>();
- for (ComponentDto viewOrSubView : dbClient.componentDao().selectEnabledViewsFromRootView(dbSession, rootViewUuid)) {
- viewAndRootViewUuidMap.put(viewOrSubView.uuid(), viewOrSubView.branchUuid());
- }
- index(dbSession, viewAndRootViewUuidMap, true, Size.REGULAR);
- }
- }
-
- /**
- * Index a single document.
- * <p/>
- * The views lookup cache will be cleared
- */
- public void index(ViewDoc viewDoc) {
- BulkIndexer bulk = new BulkIndexer(esClient, TYPE_VIEW, Size.REGULAR);
- bulk.start();
- doIndex(bulk, viewDoc, true);
- bulk.stop();
- }
-
- private void index(DbSession dbSession, Map<String, String> rootViewUuidByViewUuid, boolean needClearCache, Size bulkSize) {
- BulkIndexer bulk = new BulkIndexer(esClient, TYPE_VIEW, bulkSize);
- bulk.start();
- for (Map.Entry<String, String> entry : rootViewUuidByViewUuid.entrySet()) {
- String viewUuid = entry.getKey();
- String rootViewUuid = entry.getValue();
- List<String> projectBranchUuids = dbClient.componentDao().selectProjectBranchUuidsFromView(dbSession, viewUuid, rootViewUuid);
- doIndex(bulk, new ViewDoc()
- .setUuid(viewUuid)
- .setProjectBranchUuids(projectBranchUuids), needClearCache);
- }
- bulk.stop();
- }
-
- private void doIndex(BulkIndexer bulk, ViewDoc viewDoc, boolean needClearCache) {
- bulk.add(newIndexRequest(viewDoc));
- if (needClearCache) {
- clearLookupCache(viewDoc.uuid());
- }
- }
-
- private static IndexRequest newIndexRequest(ViewDoc doc) {
- return new IndexRequest(TYPE_VIEW.getIndex().getName())
- .id(doc.getId())
- .routing(doc.getRouting().orElse(null))
- .source(doc.getFields());
- }
-
- private void clearLookupCache(String viewUuid) {
- try {
- esClient.clearCache(new ClearIndicesCacheRequest().queryCache(true));
- } catch (Exception e) {
- throw new IllegalStateException(String.format("Unable to clear lookup cache of view '%s'", viewUuid), e);
- }
- }
-
- /**
- * This is based on the fact that a WebService is only calling {@link ViewIndexer#delete(DbSession, Collection)}
- * So the resiliency is only taking in account a deletion of view component
- * A safety check is done by not deleting any component that still exist in database.
- * This should not occur but prevent any misuse on this resiliency
- */
- @Override
- public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
- if (items.isEmpty()) {
- return new IndexingResult();
- }
-
- Set<String> viewUuids = items
- .stream()
- .map(EsQueueDto::getDocId)
- .collect(Collectors.toSet());
-
- BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items));
- bulkIndexer.start();
-
- // Safety check to remove all views that may not have been deleted
- viewUuids.removeAll(dbClient.componentDao().selectExistingUuids(dbSession, viewUuids));
- viewUuids.forEach(v -> bulkIndexer.addDeletion(TYPE_VIEW, v));
- return bulkIndexer.stop();
- }
-
- public void delete(DbSession dbSession, Collection<String> viewUuids) {
- List<EsQueueDto> items = viewUuids.stream()
- .map(l -> EsQueueDto.create(TYPE_VIEW.format(), l))
- .toList();
-
- dbClient.esQueueDao().insert(dbSession, items);
- dbSession.commit();
- index(dbSession, items);
- }
-
- private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) {
- return new BulkIndexer(esClient, TYPE_VIEW, bulkSize, listener);
- }
- }
|