From 9369c4fc6230b858596594473513f9b350d8e284 Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Fri, 22 Aug 2014 10:00:59 +0200 Subject: [PATCH] fix quality flaws --- .../org/sonar/application/DummyOkProcess.java | 22 +++- .../java/org/sonar/server/db/BaseDao.java | 3 +- .../java/org/sonar/server/db/DbClient.java | 7 + .../org/sonar/server/search/IndexQueue.java | 83 +++++++----- .../org/sonar/core/issue/db/IssueDto.java | 9 +- .../org/sonar/core/issue/db/IssueKey.java | 121 ++++++++++++++++++ 6 files changed, 202 insertions(+), 43 deletions(-) create mode 100644 sonar-core/src/main/java/org/sonar/core/issue/db/IssueKey.java diff --git a/server/process/sonar-dummy-app/src/main/java/org/sonar/application/DummyOkProcess.java b/server/process/sonar-dummy-app/src/main/java/org/sonar/application/DummyOkProcess.java index 10b95d86253..b25dfccc236 100644 --- a/server/process/sonar-dummy-app/src/main/java/org/sonar/application/DummyOkProcess.java +++ b/server/process/sonar-dummy-app/src/main/java/org/sonar/application/DummyOkProcess.java @@ -33,11 +33,16 @@ public class DummyOkProcess extends MonitoredProcess { private boolean isReady = false; private boolean isRunning = true; + private boolean isSuccess = true; - protected DummyOkProcess(Props props) throws Exception { + protected DummyOkProcess(Props props) { super(props); - File temp = File.createTempFile("hello", ".tmp"); - + try { + File.createTempFile("hello", ".tmp"); + } catch (Exception e) { + LOGGER.error("Could not create file", e); + isSuccess = false; + } } @Override @@ -64,10 +69,15 @@ public class DummyOkProcess extends MonitoredProcess { return isReady; } - public static void main(String[] args) throws Exception { + private boolean isSuccess() { + return isSuccess; + } + + public static int main(String[] args) { Props props = new Props(new Properties()); props.set(MonitoredProcess.NAME_PROPERTY, DummyOkProcess.class.getSimpleName()); - new DummyOkProcess(props).start(); - System.exit(1); + DummyOkProcess process = new DummyOkProcess(props); + process.start(); + return (process.isSuccess()) ? 1 : 0; } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/BaseDao.java b/server/sonar-server/src/main/java/org/sonar/server/db/BaseDao.java index cc5ff34ea8e..9724eda67f3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/BaseDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/BaseDao.java @@ -219,8 +219,7 @@ public abstract class BaseDao, K extends Serializable> imple session.enqueue(new UpsertDto(getIndexType(), item)); } } catch (Exception e) { - e.printStackTrace(); - throw new IllegalStateException("Fail to insert item in db: " + item, e.getCause()); + throw new IllegalStateException("Fail to insert item in db: " + item, e); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java b/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java index 2b7b1351f24..846357d0bc4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java @@ -34,6 +34,7 @@ import org.sonar.core.user.AuthorizationDao; import org.sonar.core.user.UserDao; import org.sonar.server.activity.db.ActivityDao; import org.sonar.server.component.persistence.ComponentDao; +import org.sonar.server.issue.db.IssueDao; import org.sonar.server.measure.persistence.MeasureDao; import org.sonar.server.measure.persistence.MetricDao; import org.sonar.server.qualityprofile.db.ActiveRuleDao; @@ -61,6 +62,7 @@ public class DbClient implements ServerComponent { private final ActivityDao activityDao; private final AuthorizationDao authorizationDao; private final UserDao userDao; + private final IssueDao issueDao; public DbClient(Database db, MyBatis myBatis, DaoComponent... daoComponents) { this.db = db; @@ -83,6 +85,7 @@ public class DbClient implements ServerComponent { activityDao = getDao(map, ActivityDao.class); authorizationDao = getDao(map, AuthorizationDao.class); userDao = getDao(map, UserDao.class); + issueDao = getDao(map, IssueDao.class); } public Database database() { @@ -101,6 +104,10 @@ public class DbClient implements ServerComponent { return activeRuleDao; } + public IssueDao issueDao() { + return issueDao; + } + public QualityProfileDao qualityProfileDao() { return qualityProfileDao; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java index a8003978d94..b159aae781a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexQueue.java @@ -20,6 +20,8 @@ package org.sonar.server.search; import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; @@ -54,7 +56,7 @@ public class IndexQueue extends LinkedBlockingQueue private static final Logger LOGGER = LoggerFactory.getLogger(IndexQueue.class); private static final Integer DEFAULT_QUEUE_SIZE = 200; - private static final int TIMEOUT = 30000; + private static final Integer CONCURRENT_NORMALIZATION_FACTOR = 5; public IndexQueue(Settings settings, SearchClient searchClient, ComponentContainer container) { super(DEFAULT_QUEUE_SIZE); @@ -71,28 +73,62 @@ public class IndexQueue extends LinkedBlockingQueue } try { - long normTime = System.currentTimeMillis(); - BulkRequestBuilder bulkRequestBuilder = new BulkRequestBuilder(searchClient); Map indexes = getIndexMap(); Set indices = new HashSet(); for (IndexActionRequest action : actions) { Index index = indexes.get(action.getIndexType()); action.setIndex(index); - indices.add(index.getIndexName()); + if (action.needsRefresh()) { + indices.add(index.getIndexName()); + } } - ExecutorService executorService = Executors.newFixedThreadPool(10); + BulkRequestBuilder bulkRequestBuilder = new BulkRequestBuilder(searchClient); - // Do we need to refresh - boolean requiresRefresh = false; - for (IndexActionRequest action : actions) { - if (action.needsRefresh()) { - requiresRefresh = true; - break; - } + long normTime = executeNormalization(bulkRequestBuilder, actions); + + //execute the request + long indexTime = System.currentTimeMillis(); + BulkResponse response = searchClient.execute(bulkRequestBuilder.setRefresh(false)); + indexTime = System.currentTimeMillis() - indexTime; + + long refreshTime = this.refreshRequiredIndex(indices); + + LOGGER.debug("-- submitted {} items with {}ms in normalization, {}ms indexing and {}ms refresh({}). Total: {}ms", + bulkRequestBuilder.numberOfActions(), normTime, indexTime, refreshTime, indices, (normTime + indexTime + refreshTime)); + + if (response.hasFailures()) { + throw new IllegalStateException("Errors while indexing stack: " + response.buildFailureMessage()); } - //invokeAll() blocks until ALL tasks submitted to executor complete + } catch (Exception e) { + LOGGER.error("Could not commit to ElasticSearch", e); + } + } + + + private long refreshRequiredIndex(Set indices) { + + long refreshTime = System.currentTimeMillis(); + if (!indices.isEmpty()) { + RefreshRequestBuilder refreshRequest = searchClient.admin().indices() + .prepareRefresh(indices.toArray(new String[indices.size()])) + .setForce(false); + + RefreshResponse refreshResponse = searchClient.execute(refreshRequest); + + if (refreshResponse.getFailedShards() > 0) { + LOGGER.warn("{} Shard(s) did not refresh", refreshResponse.getFailedShards()); + } + } + return System.currentTimeMillis() - refreshTime; + } + + private long executeNormalization(BulkRequestBuilder bulkRequestBuilder, List actions) { + long normTime = System.currentTimeMillis(); + ExecutorService executorService = Executors.newFixedThreadPool(CONCURRENT_NORMALIZATION_FACTOR); + //invokeAll() blocks until ALL tasks submitted to executor complete + try { for (Future> updateRequests : executorService.invokeAll(actions)) { for (ActionRequest update : updateRequests.get()) { if (UpdateRequest.class.isAssignableFrom(update.getClass())) { @@ -104,26 +140,11 @@ public class IndexQueue extends LinkedBlockingQueue } } } - executorService.shutdown(); - normTime = System.currentTimeMillis() - normTime; - - //execute the request - long indexTime = System.currentTimeMillis(); - BulkResponse response = searchClient.execute(bulkRequestBuilder.setRefresh(false)); - indexTime = System.currentTimeMillis() - indexTime; - - long refreshTime = System.currentTimeMillis(); - if (requiresRefresh) { - searchClient.admin().indices().prepareRefresh(indices.toArray(new String[indices.size()])).setForce(false).get(); - } - refreshTime = System.currentTimeMillis() - refreshTime; - - LOGGER.debug("-- submitted {} items with {}ms in normalization, {}ms indexing and {}ms refresh({}). Total: {}ms", - bulkRequestBuilder.numberOfActions(), normTime, indexTime, refreshTime, indices, (normTime + indexTime + refreshTime)); - } catch (Exception e) { - e.printStackTrace(); + throw new IllegalStateException("Could not execute normalization for stack", e); } + executorService.shutdown(); + return System.currentTimeMillis() - normTime; } private Map getIndexMap() { diff --git a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java index 858a1b7c8ff..ddb98282cdc 100644 --- a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java +++ b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueDto.java @@ -37,7 +37,7 @@ import java.util.Date; /** * @since 3.6 */ -public final class IssueDto extends Dto implements Serializable { +public final class IssueDto extends Dto implements Serializable { private Long id; @@ -76,16 +76,17 @@ public final class IssueDto extends Dto implements Serializable { private String componentKey; private String rootComponentKey; - @Override - public String getKey() { - return kee; + public IssueKey getKey() { + return IssueKey.of(ruleKey, ruleRepo, rootComponentKey, componentKey); } + @Deprecated public Long getId() { return id; } + @Deprecated public IssueDto setId(@Nullable Long id) { this.id = id; return this; diff --git a/sonar-core/src/main/java/org/sonar/core/issue/db/IssueKey.java b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueKey.java new file mode 100644 index 00000000000..b9345a11731 --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/issue/db/IssueKey.java @@ -0,0 +1,121 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.issue.db; + +import com.google.common.base.Preconditions; +import org.sonar.api.rule.RuleKey; + +import java.io.Serializable; + +public class IssueKey implements Serializable { + + private final RuleKey ruleKey; + private final String projectKey, componentKey; + + protected IssueKey(RuleKey ruleKey, String projectKey, String componentKey) { + Preconditions.checkNotNull(ruleKey, "RuleKey is missing"); + Preconditions.checkNotNull(projectKey, "Project is missing"); + this.ruleKey = ruleKey; + this.projectKey = projectKey; + this.componentKey = componentKey; + } + + + /** + * Create a key. Parameters are NOT null. + */ + public static IssueKey of(String ruleKey, String ruleRepo, String rootComponentKey, String componentKey) { + return new IssueKey(RuleKey.of(ruleRepo, ruleKey), rootComponentKey, componentKey); + } + + /** + * Create a key from a string representation (see {@link #toString()}. An {@link IllegalArgumentException} is raised + * if the format is not valid. + */ + public static IssueKey parse(String s) { + Preconditions.checkArgument(s.split(":").length >= 4, "Bad format of issueKey key: " + s); +// int semiColonPos = s.indexOf(":"); +// String key = s.substring(0, semiColonPos); +// String ruleKey = s.substring(semiColonPos + 1); +// return IssueKey.of(key, RuleKey.parse(ruleKey)); + return null; + } + + /** + * Never null + */ + public RuleKey ruleKey() { + return ruleKey; + } + + /** + * Never null + */ + public String projectKey() { + return projectKey; + } + + /** + * Never null + */ + public String componentKey() { + return componentKey; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IssueKey key = (IssueKey) o; + if (!ruleKey.equals(key.ruleKey)) { + return false; + } + if (!projectKey.equals(key.projectKey)) { + return false; + } + if (!componentKey.equals(key.componentKey)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int result = ruleKey.hashCode(); + result = 31 * result + projectKey.hashCode(); + result = 31 * result + componentKey.hashCode(); + return result; + } + + + /** + * Format is "qprofile:rule:project:component", for example "12345:squid:AvoidCycle" + */ + @Override + public String toString() { + return String.format("%s:%s:%s", ruleKey.toString(), projectKey(), componentKey()); + } + +} -- 2.39.5