import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.stream.IntStream;
+import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
+import org.elasticsearch.search.aggregations.bucket.nested.Nested;
import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
+import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.server.es.DefaultIndexSettingsElement;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
import static org.elasticsearch.search.aggregations.AggregationBuilders.filters;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
import static org.elasticsearch.search.sort.SortOrder.ASC;
import static org.elasticsearch.search.sort.SortOrder.DESC;
import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY;
import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
import static org.sonar.api.measures.CoreMetrics.DUPLICATED_LINES_DENSITY_KEY;
+import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
import static org.sonar.api.measures.CoreMetrics.NEW_COVERAGE_KEY;
import static org.sonar.api.measures.CoreMetrics.NEW_DUPLICATED_LINES_DENSITY_KEY;
import static org.sonar.api.measures.CoreMetrics.SECURITY_RATING_KEY;
import static org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY;
import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars;
+import static org.sonar.server.es.EsUtils.termsToMap;
import static org.sonar.server.measure.index.ProjectMeasuresDoc.QUALITY_GATE_STATUS;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY;
import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_NAME;
import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_LANGUAGES;
import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_TAGS;
+import static org.sonarqube.ws.client.project.ProjectsWsParameters.MAX_PAGE_SIZE;
public class ProjectMeasuresIndex {
private static final Double[] LINES_THRESHOLDS = new Double[] {1_000d, 10_000d, 100_000d, 500_000d};
private static final Double[] COVERAGE_THRESHOLDS = new Double[] {30d, 50d, 70d, 80d};
private static final Double[] DUPLICATIONS_THRESHOLDS = new Double[] {3d, 5d, 10d, 20d};
-
+
private static final String FIELD_MEASURES_KEY = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY;
private static final String FIELD_MEASURES_VALUE = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE;
return new SearchIdResult<>(requestBuilder.get(), id -> id);
}
+ public ProjectMeasuresStatistics searchTelemetryStatistics() {
+ SearchRequestBuilder request = client
+ .prepareSearch(INDEX_TYPE_PROJECT_MEASURES)
+ .setFetchSource(false)
+ .setSize(0);
+
+ BoolQueryBuilder esFilter = boolQuery();
+ request.setQuery(esFilter);
+ request.addAggregation(AggregationBuilders.terms(FIELD_LANGUAGES)
+ .field(FIELD_LANGUAGES)
+ .size(MAX_PAGE_SIZE)
+ .minDocCount(1)
+ .order(Terms.Order.count(false)));
+ Stream.of(LINES_KEY, NCLOC_KEY)
+ .forEach(metric -> request.addAggregation(AggregationBuilders.nested(metric, FIELD_MEASURES)
+ .subAggregation(AggregationBuilders.filter(metric + "_filter", termQuery(FIELD_MEASURES_KEY, metric))
+ .subAggregation(sum(metric + "_filter_sum").field(FIELD_MEASURES_VALUE)))));
+
+ ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder();
+
+ SearchResponse response = request.get();
+ statistics.setProjectCount(response.getHits().getTotalHits());
+ Stream.of(LINES_KEY, NCLOC_KEY)
+ .map(metric -> (Nested) response.getAggregations().get(metric))
+ .map(nested -> (Filter) nested.getAggregations().get(nested.getName() + "_filter"))
+ .map(filter -> (Sum) filter.getAggregations().get(filter.getName() + "_sum"))
+ .forEach(sum -> {
+ String metric = sum.getName().replace("_filter_sum", "");
+ long value = Math.round(sum.getValue());
+ statistics.setSum(metric, value);
+ });
+ statistics.setProjectLanguageDistribution(termsToMap(response.getAggregations().get(FIELD_LANGUAGES)));
+
+ return statistics.build();
+ }
+
private static void addSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder) {
String sort = query.getSort();
if (SORT_BY_NAME.equals(sort)) {
}
private static void addLanguagesFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder) {
- Optional<Set<String>> languages = query.getLanguages();
- esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_LANGUAGES, FILTER_LANGUAGES, languages.isPresent() ? languages.get().toArray() : new Object[] {}));
+ esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_LANGUAGES, FILTER_LANGUAGES, query.getLanguages().map(Set::toArray).orElseGet(() -> new Object[] {})));
}
private static void addTagsFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder) {
- Optional<Set<String>> tags = query.getTags();
- esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_TAGS, FILTER_TAGS, tags.isPresent() ? tags.get().toArray() : new Object[] {}));
+ esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_TAGS, FILTER_TAGS, query.getTags().map(Set::toArray).orElseGet(() -> new Object[] {})));
}
private static void addFacets(SearchRequestBuilder esSearch, SearchOptions options, Map<String, QueryBuilder> filters, ProjectMeasuresQuery query) {
private static AbstractAggregationBuilder createRatingFacet(String metricKey) {
return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES)
.subAggregation(
- AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey))
- .subAggregation(filters(metricKey,
- new KeyedFilter("1", termQuery(FIELD_MEASURES_VALUE, 1d)),
- new KeyedFilter("2", termQuery(FIELD_MEASURES_VALUE, 2d)),
- new KeyedFilter("3", termQuery(FIELD_MEASURES_VALUE, 3d)),
- new KeyedFilter("4", termQuery(FIELD_MEASURES_VALUE, 4d)),
- new KeyedFilter("5", termQuery(FIELD_MEASURES_VALUE, 5d)))));
+ AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey))
+ .subAggregation(filters(metricKey,
+ new KeyedFilter("1", termQuery(FIELD_MEASURES_VALUE, 1d)),
+ new KeyedFilter("2", termQuery(FIELD_MEASURES_VALUE, 2d)),
+ new KeyedFilter("3", termQuery(FIELD_MEASURES_VALUE, 3d)),
+ new KeyedFilter("4", termQuery(FIELD_MEASURES_VALUE, 4d)),
+ new KeyedFilter("5", termQuery(FIELD_MEASURES_VALUE, 5d)))));
}
private static AbstractAggregationBuilder createQualityGateFacet() {
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.measure.index;
+
+import java.util.Map;
+
+import static java.util.Objects.requireNonNull;
+import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
+import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
+
+public class ProjectMeasuresStatistics {
+ private final long projectCount;
+ private final long lines;
+ private final long ncloc;
+ private final Map<String, Long> projectLanguageDistribution;
+
+ private ProjectMeasuresStatistics(Builder builder) {
+ projectCount = builder.projectCount;
+ lines = builder.lines;
+ ncloc = builder.ncloc;
+ projectLanguageDistribution = builder.projectLanguageDistribution;
+ }
+
+ public long getProjectCount() {
+ return projectCount;
+ }
+
+ public long getLines() {
+ return lines;
+ }
+
+ public long getNcloc() {
+ return ncloc;
+ }
+
+ public Map<String, Long> getProjectLanguageDistribution() {
+ return projectLanguageDistribution;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private Map<String, Long> projectLanguageDistribution;
+ private Long projectCount;
+ private Long lines;
+ private Long ncloc;
+
+ private Builder() {
+ // enforce static factory method
+ }
+
+ public Builder setProjectCount(long projectCount) {
+ this.projectCount = projectCount;
+ return this;
+ }
+
+ public Builder setSum(String metric, long value) {
+ switch (metric) {
+ case LINES_KEY:
+ this.lines = value;
+ break;
+ case NCLOC_KEY:
+ this.ncloc = value;
+ break;
+ default:
+ throw new IllegalStateException("Metric not supported: " + metric);
+ }
+ return this;
+ }
+
+ public void setProjectLanguageDistribution(Map<String, Long> projectLanguageDistribution) {
+ this.projectLanguageDistribution = projectLanguageDistribution;
+ }
+
+ public ProjectMeasuresStatistics build() {
+ requireNonNull(projectCount);
+ requireNonNull(lines);
+ requireNonNull(ncloc);
+ requireNonNull(projectLanguageDistribution);
+ return new ProjectMeasuresStatistics(this);
+ }
+ }
+}
import org.sonar.api.utils.log.Loggers;
import org.sonar.api.utils.text.JsonWriter;
import org.sonar.core.platform.PluginRepository;
+import org.sonar.server.es.SearchOptions;
+import org.sonar.server.measure.index.ProjectMeasuresIndex;
+import org.sonar.server.measure.index.ProjectMeasuresStatistics;
import org.sonar.server.property.InternalProperties;
+import org.sonar.server.user.index.UserIndex;
+import org.sonar.server.user.index.UserQuery;
+import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
+import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
import static org.sonar.api.utils.DateUtils.formatDate;
import static org.sonar.api.utils.DateUtils.parseDate;
import static org.sonar.core.config.TelemetryProperties.PROP_ENABLE;
private final Server server;
private final PluginRepository pluginRepository;
private final System2 system2;
+ private final UserIndex userIndex;
+ private final ProjectMeasuresIndex projectMeasuresIndex;
private ScheduledExecutorService executorService;
public TelemetryDaemon(TelemetryClient telemetryClient, Configuration config, InternalProperties internalProperties, Server server, PluginRepository pluginRepository,
- System2 system2) {
+ System2 system2, UserIndex userIndex, ProjectMeasuresIndex projectMeasuresIndex) {
this.telemetryClient = telemetryClient;
this.config = config;
this.internalProperties = internalProperties;
this.server = server;
this.pluginRepository = pluginRepository;
this.system2 = system2;
+ this.userIndex = userIndex;
+ this.projectMeasuresIndex = projectMeasuresIndex;
}
@Override
writer.prop(plugin.getKey(), version);
});
writer.endObject();
+ long userCount = userIndex.search(UserQuery.builder().build(), new SearchOptions().setLimit(1)).getTotal();
+ writer.prop("userCount", userCount);
+ ProjectMeasuresStatistics statistics = projectMeasuresIndex.searchTelemetryStatistics();
+ writer.prop("projectCount", statistics.getProjectCount());
+ writer.prop(LINES_KEY, statistics.getLines());
+ writer.prop(NCLOC_KEY, statistics.getNcloc());
+ writer.name("projectLanguageDistribution");
+ writer.valueObject(statistics.getProjectLanguageDistribution());
writer.endObject();
}
telemetryClient.upload(json.toString());
assertThat(result).isEmpty();
}
+ @Test
+ public void search_statistics() {
+ es.putDocuments(INDEX_TYPE_PROJECT_MEASURES,
+ newDoc("lines", 10, "ncloc", 20, "coverage", 80).setLanguages(Arrays.asList("java", "cs", "js")),
+ newDoc("lines", 20, "ncloc", 30, "coverage", 80).setLanguages(Arrays.asList("java", "python", "kotlin")));
+
+ ProjectMeasuresStatistics result = underTest.searchTelemetryStatistics();
+
+ assertThat(result.getProjectCount()).isEqualTo(2);
+ assertThat(result.getLines()).isEqualTo(30);
+ assertThat(result.getNcloc()).isEqualTo(50);
+ assertThat(result.getProjectLanguageDistribution()).containsOnly(
+ entry("java", 2L), entry("cs", 1L), entry("js", 1L), entry("python", 1L), entry("kotlin", 1L));
+ }
+
@Test
public void fail_if_page_size_greater_than_500() {
expectedException.expect(IllegalArgumentException.class);
package org.sonar.server.telemetry;
+import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.sonar.api.config.Configuration;
import org.sonar.api.config.PropertyDefinitions;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.internal.TestSystem2;
import org.sonar.core.config.TelemetryProperties;
import org.sonar.core.platform.PluginInfo;
import org.sonar.core.platform.PluginRepository;
+import org.sonar.server.es.EsTester;
+import org.sonar.server.measure.index.ProjectMeasuresDoc;
+import org.sonar.server.measure.index.ProjectMeasuresIndex;
+import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
import org.sonar.server.property.InternalProperties;
import org.sonar.server.property.MapInternalProperties;
+import org.sonar.server.tester.UserSessionRule;
+import org.sonar.server.user.index.UserDoc;
+import org.sonar.server.user.index.UserIndex;
+import org.sonar.server.user.index.UserIndexDefinition;
import org.sonar.updatecenter.common.Version;
import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
private static final long ONE_HOUR = 60 * 60 * 1_000L;
private static final long ONE_DAY = 24 * ONE_HOUR;
+ private static final Configuration emptyConfig = new MapSettings().asConfig();
+
+ @Rule
+ public UserSessionRule userSession = UserSessionRule.standalone();
+ @Rule
+ public EsTester es = new EsTester(new UserIndexDefinition(emptyConfig), new ProjectMeasuresIndexDefinition(emptyConfig));
private TelemetryClient client = mock(TelemetryClient.class);
private InternalProperties internalProperties = new MapInternalProperties();
settings = new MapSettings(new PropertyDefinitions(TelemetryProperties.all()));
system2.setNow(System.currentTimeMillis());
- underTest = new TelemetryDaemon(client, settings.asConfig(), internalProperties, server, pluginRepository, system2);
+ underTest = new TelemetryDaemon(client, settings.asConfig(), internalProperties, server, pluginRepository, system2, new UserIndex(es.client()),
+ new ProjectMeasuresIndex(es.client(), null));
}
@Test
server.setVersion(version);
List<PluginInfo> plugins = Arrays.asList(newPlugin("java", "4.12.0.11033"), newPlugin("scmgit", "1.2"), new PluginInfo("other"));
when(pluginRepository.getPluginInfos()).thenReturn(plugins);
+ es.putDocuments(UserIndexDefinition.INDEX_TYPE_USER,
+ new UserDoc().setLogin(randomAlphanumeric(30)).setActive(true),
+ new UserDoc().setLogin(randomAlphanumeric(30)).setActive(true),
+ new UserDoc().setLogin(randomAlphanumeric(30)).setActive(true),
+ new UserDoc().setLogin(randomAlphanumeric(30)).setActive(false));
+ es.putDocuments(ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES,
+ new ProjectMeasuresDoc().setId(randomAlphanumeric(20))
+ .setMeasures(Arrays.asList(newMeasure("lines", 200), newMeasure("ncloc", 100), newMeasure("coverage", 80)))
+ .setLanguages(Arrays.asList("java", "js")),
+ new ProjectMeasuresDoc().setId(randomAlphanumeric(20))
+ .setMeasures(Arrays.asList(newMeasure("lines", 300), newMeasure("ncloc", 200), newMeasure("coverage", 80)))
+ .setLanguages(Arrays.asList("java", "kotlin")));
+
underTest.start();
ArgumentCaptor<String> jsonCaptor = ArgumentCaptor.forClass(String.class);
settings.setProperty(PROP_FREQUENCY, "1");
long today = parseDate("2017-08-01").getTime();
system2.setNow(today + 15 * ONE_HOUR);
- long now = system2.now();
- long sevenDaysAgo = now - (ONE_DAY * 7L);
+ long sevenDaysAgo = today - (ONE_DAY * 7L);
internalProperties.write(I_PROP_LAST_PING, String.valueOf(sevenDaysAgo));
underTest.start();
underTest.start();
underTest.start();
-
verify(client, timeout(1_000).never()).upload(anyString());
verify(client, timeout(1_000).times(1)).optOut(anyString());
}
return new PluginInfo(key)
.setVersion(Version.create(version));
}
+
+ private static Map<String, Object> newMeasure(String key, Object value) {
+ return ImmutableMap.of("key", key, "value", value);
+ }
}
"java": "4.12.0.11033",
"scmgit": "1.2",
"other": "undefined"
+ },
+ "userCount": 3,
+ "projectCount": 2,
+ "lines": 500,
+ "ncloc": 300,
+ "projectLanguageDistribution": {
+ "java": 2,
+ "kotlin": 1,
+ "js": 1
}
}
package org.sonarqube.tests.telemetry;
import com.sonar.orchestrator.Orchestrator;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.core.HttpHeaders;
import okhttp3.mockwebserver.MockWebServer;
import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
import static org.assertj.core.api.Assertions.assertThat;
+import static util.ItUtils.jsonToMap;
import static util.ItUtils.xooPlugin;
public class TelemetryTest {
RecordedRequest request = server.takeRequest(1, TimeUnit.SECONDS);
assertThat(request.getMethod()).isEqualTo("POST");
- assertThat(request.getBody().readUtf8()).contains(serverId);
assertThat(request.getHeader(HttpHeaders.USER_AGENT)).contains("SonarQube");
+ String body = request.getBody().readUtf8();
+ System.out.println(body);
+ Map<String, Object> json = jsonToMap(body);
+ assertThat(json.get("id")).isEqualTo(serverId);
+ assertThat(json.get("ncloc")).isEqualTo(0.0d);
+ assertThat(json.get("lines")).isEqualTo(0.0d);
+ assertThat(((Map)json.get("plugins")).keySet()).contains("xoo");
orchestrator.stop();
}
projects / sonarqube.git / commitdiff