diff options
author | alain <108417558+alain-kermis-sonarsource@users.noreply.github.com> | 2022-08-30 17:00:34 +0200 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2022-09-05 20:02:56 +0000 |
commit | cd01dd4e2a883177d515b4f71554ee90e449b742 (patch) | |
tree | 1dbcee5083bb35765b17bb9ddd1ce07b349ab401 | |
parent | 3aae5514421557500b047eab24d728979fe24f25 (diff) | |
download | sonarqube-cd01dd4e2a883177d515b4f71554ee90e449b742.tar.gz sonarqube-cd01dd4e2a883177d515b4f71554ee90e449b742.zip |
SONAR-17195 Add new telemetry fields
35 files changed, 967 insertions, 233 deletions
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java index 4327429abc8..02a2576417d 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java @@ -35,6 +35,7 @@ import org.apache.ibatis.session.SqlSessionFactoryBuilder; import org.apache.ibatis.session.TransactionIsolationLevel; import org.sonar.db.alm.pat.AlmPatMapper; import org.sonar.db.alm.setting.AlmSettingMapper; +import org.sonar.db.alm.setting.ProjectAlmKeyAndProject; import org.sonar.db.alm.setting.ProjectAlmSettingMapper; import org.sonar.db.audit.AuditMapper; import org.sonar.db.ce.CeActivityMapper; @@ -45,6 +46,7 @@ import org.sonar.db.ce.CeTaskCharacteristicMapper; import org.sonar.db.ce.CeTaskInputMapper; import org.sonar.db.ce.CeTaskMessageMapper; import org.sonar.db.component.AnalysisPropertiesMapper; +import org.sonar.db.component.AnalysisPropertyValuePerProject; import org.sonar.db.component.ApplicationProjectsMapper; import org.sonar.db.component.BranchMapper; import org.sonar.db.component.ComponentDto; @@ -54,7 +56,7 @@ import org.sonar.db.component.ComponentMapper; import org.sonar.db.component.ComponentWithModuleUuidDto; import org.sonar.db.component.FilePathWithHashDto; import org.sonar.db.component.KeyWithUuidDto; -import org.sonar.db.component.ProjectCountPerAnalysisPropertyValue; +import org.sonar.db.component.PrAndBranchCountByProjectDto; import org.sonar.db.component.ProjectLinkMapper; import org.sonar.db.component.ResourceDto; import org.sonar.db.component.ScrapAnalysisPropertyDto; @@ -80,6 +82,7 @@ import org.sonar.db.measure.LargestBranchNclocDto; import org.sonar.db.measure.LiveMeasureMapper; import org.sonar.db.measure.MeasureDto; import org.sonar.db.measure.MeasureMapper; +import org.sonar.db.measure.ProjectMeasureDto; import org.sonar.db.metric.MetricMapper; import org.sonar.db.newcodeperiod.NewCodePeriodMapper; import org.sonar.db.notification.NotificationQueueDto; @@ -154,6 +157,7 @@ import org.sonar.db.user.UserDto; import org.sonar.db.user.UserGroupDto; import org.sonar.db.user.UserGroupMapper; import org.sonar.db.user.UserMapper; +import org.sonar.db.user.UserTelemetryDto; import org.sonar.db.user.UserTokenCount; import org.sonar.db.user.UserTokenDto; import org.sonar.db.user.UserTokenMapper; @@ -218,8 +222,11 @@ public class MyBatis { confBuilder.loadAlias("ProjectQgateAssociation", ProjectQgateAssociationDto.class); confBuilder.loadAlias("Project", ProjectDto.class); confBuilder.loadAlias("ProjectBadgeToken", ProjectBadgeTokenDto.class); - confBuilder.loadAlias("ProjectCountPerAnalysisPropertyValue", ProjectCountPerAnalysisPropertyValue.class); + confBuilder.loadAlias("AnalysisPropertyValuePerProject", AnalysisPropertyValuePerProject.class); + confBuilder.loadAlias("ProjectAlmKeyAndProject", ProjectAlmKeyAndProject.class); + confBuilder.loadAlias("PrAndBranchCountByProjectDto", PrAndBranchCountByProjectDto.class); confBuilder.loadAlias("ProjectMapping", ProjectMappingDto.class); + confBuilder.loadAlias("ProjectMeasure", ProjectMeasureDto.class); confBuilder.loadAlias("PurgeableAnalysis", PurgeableAnalysisDto.class); confBuilder.loadAlias("PushEvent", PushEventDto.class); confBuilder.loadAlias("QualityGateCondition", QualityGateConditionDto.class); @@ -230,11 +237,12 @@ public class MyBatis { confBuilder.loadAlias("ScrapProperty", ScrapPropertyDto.class); confBuilder.loadAlias("ScrapAnalysisProperty", ScrapAnalysisPropertyDto.class); confBuilder.loadAlias("Snapshot", SnapshotDto.class); + confBuilder.loadAlias("User", UserDto.class); confBuilder.loadAlias("UserGroup", UserGroupDto.class); confBuilder.loadAlias("UserPermission", UserPermissionDto.class); - confBuilder.loadAlias("UserTokenCount", UserTokenCount.class); + confBuilder.loadAlias("UserTelemetry", UserTelemetryDto.class); confBuilder.loadAlias("UserToken", UserTokenDto.class); - confBuilder.loadAlias("User", UserDto.class); + confBuilder.loadAlias("UserTokenCount", UserTokenCount.class); confBuilder.loadAlias("UuidWithProjectUuid", UuidWithProjectUuidDto.class); confBuilder.loadAlias("ViewsSnapshot", ViewsSnapshotDto.class); confExtensions.forEach(ext -> ext.loadAliases(confBuilder::loadAlias)); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmKeyAndProject.java b/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmKeyAndProject.java new file mode 100644 index 00000000000..be435294c7e --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmKeyAndProject.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.alm.setting; + +public class ProjectAlmKeyAndProject { + + private String projectUuid; + private String almId; + private String url; + + public ProjectAlmKeyAndProject() { + // keep empty + } + + public String getProjectUuid() { + return projectUuid; + } + + public ProjectAlmKeyAndProject setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; + return this; + } + + public String getAlmId() { + return almId; + } + + public ProjectAlmKeyAndProject setAlmId(String almId) { + this.almId = almId; + return this; + } + + public String getUrl() { + return url; + } + + public ProjectAlmKeyAndProject setUrl(String url) { + this.url = url; + return this; + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingDao.java index c8c44cf9dac..f6c7867e7fb 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingDao.java @@ -101,4 +101,8 @@ public class ProjectAlmSettingDao implements Dao { public List<ProjectAlmSettingDto> selectByAlmSettingAndRepos(DbSession dbSession, AlmSettingDto almSettingDto, Set<String> almRepos) { return executeLargeInputs(almRepos, repos -> getMapper(dbSession).selectByAlmSettingAndRepos(almSettingDto.getUuid(), repos)); } + + public List<ProjectAlmKeyAndProject> selectAlmTypeAndUrlByProject(DbSession dbSession) { + return getMapper(dbSession).selectAlmTypeAndUrlByProject(); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingMapper.java index 98c7478d6e7..6e8425501b0 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/alm/setting/ProjectAlmSettingMapper.java @@ -40,4 +40,6 @@ public interface ProjectAlmSettingMapper { List<ProjectAlmSettingDto> selectByAlmSettingAndSlugs(@Param("almSettingUuid") String almSettingUuid, @Param("slugs") List<String> slugs); List<ProjectAlmSettingDto> selectByAlmSettingAndRepos(@Param("almSettingUuid") String almSettingUuid, @Param("repos") List<String> repos); + + List<ProjectAlmKeyAndProject> selectAlmTypeAndUrlByProject(); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesDao.java index 72dde4807dd..8227d65b1c6 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesDao.java @@ -71,8 +71,8 @@ public class AnalysisPropertiesDao implements Dao { } } - public List<ProjectCountPerAnalysisPropertyValue> selectProjectCountPerAnalysisPropertyValueInLastAnalysis(DbSession session, String analysisPropertyKey) { - return getMapper(session).selectProjectCountPerAnalysisPropertyValueInLastAnalysis(analysisPropertyKey); + public List<AnalysisPropertyValuePerProject> selectAnalysisPropertyValueInLastAnalysisPerProject(DbSession session, String analysisPropertyKey) { + return getMapper(session).selectAnalysisPropertyValueInLastAnalysisPerProject(analysisPropertyKey); } private static boolean mustBeStoredInClob(String value) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesMapper.java index 34839273c96..e83046d8306 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertiesMapper.java @@ -33,7 +33,7 @@ public interface AnalysisPropertiesMapper { void insertAsText(@Param("analysisPropertyDto") AnalysisPropertyDto analysisPropertyDto, @Param("createdAt") long createdAt); - List<ProjectCountPerAnalysisPropertyValue> selectProjectCountPerAnalysisPropertyValueInLastAnalysis(@Param("analysisPropertyKey") String analysisPropertyKey); + List<AnalysisPropertyValuePerProject> selectAnalysisPropertyValueInLastAnalysisPerProject(@Param("analysisPropertyKey") String analysisPropertyKey); List<AnalysisPropertyDto> selectByKeyAnAnalysisUuids(@Param("analysisUuids") Collection<String> analysisUuids, @Param("analysisPropertyKey") String analysisPropertyKey); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ProjectCountPerAnalysisPropertyValue.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertyValuePerProject.java index 21f4de3758c..ca6d73a64aa 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/ProjectCountPerAnalysisPropertyValue.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/AnalysisPropertyValuePerProject.java @@ -19,11 +19,11 @@ */ package org.sonar.db.component; -public class ProjectCountPerAnalysisPropertyValue { +public class AnalysisPropertyValuePerProject { + private String projectUuid; private String propertyValue; - private Long count; - public ProjectCountPerAnalysisPropertyValue() { + public AnalysisPropertyValuePerProject() { //nothing to do here } @@ -35,11 +35,11 @@ public class ProjectCountPerAnalysisPropertyValue { this.propertyValue = propertyValue; } - public Long getCount() { - return count; + public String getProjectUuid() { + return projectUuid; } - public void setCount(Long count) { - this.count = count; + public void setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java index f03eea0479c..9d6cc0f6d29 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java @@ -101,6 +101,10 @@ public class BranchDao implements Dao { return mapper(dbSession).selectByProjectUuid(project.getUuid()); } + public List<PrAndBranchCountByProjectDto> countPrAndBranchByProjectUuid(DbSession dbSession){ + return mapper(dbSession).countPrAndBranchByProjectUuid(); + } + public List<BranchDto> selectByUuids(DbSession session, Collection<String> uuids) { if (uuids.isEmpty()) { return emptyList(); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java index 166765d4994..a1793e1d7dd 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java @@ -47,6 +47,8 @@ public interface BranchMapper { Collection<BranchDto> selectByProjectUuid(@Param("projectUuid") String projectUuid); + List<PrAndBranchCountByProjectDto> countPrAndBranchByProjectUuid(); + List<BranchDto> selectByBranchKeys(@Param("branchKeyByProjectUuid") Map<String, String> branchKeyByProjectUuid); List<BranchDto> selectByUuids(@Param("uuids") Collection<String> uuids); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/PrAndBranchCountByProjectDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/PrAndBranchCountByProjectDto.java new file mode 100644 index 00000000000..ef9149bfb8c --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/PrAndBranchCountByProjectDto.java @@ -0,0 +1,51 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.component; + +public class PrAndBranchCountByProjectDto { + + private String projectUuid = null; + private Long pullRequest = null; + private Long branch = null; + + public String getProjectUuid() { + return projectUuid; + } + + public void setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; + } + + public Long getPullRequest() { + return pullRequest; + } + + public void setPullRequest(Long pullRequest) { + this.pullRequest = pullRequest; + } + + public Long getBranch() { + return branch; + } + + public void setBranch(Long branch) { + this.branch = branch; + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java index ddac96633da..d6b8adb6a02 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java @@ -75,4 +75,9 @@ public class MeasureDao implements Dao { private static MeasureMapper mapper(DbSession session) { return session.getMapper(MeasureMapper.class); } + + public List<ProjectMeasureDto> selectLastMeasureForAllProjects(DbSession session, String metricKey) { + return mapper(session).selectLastMeasureForAllProjects(metricKey); + + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java index 68829023849..9f80836c15b 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java @@ -41,4 +41,6 @@ public interface MeasureMapper { List<MeasureDto> selectPastMeasuresOnSeveralAnalyses(@Param("query") PastMeasureQuery query); void insert(MeasureDto measureDto); + + List<ProjectMeasureDto> selectLastMeasureForAllProjects(@Param("metricKey") String metricKey); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasureDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasureDto.java new file mode 100644 index 00000000000..8d9bb72cf8e --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasureDto.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.measure; + +public class ProjectMeasureDto { + + private String projectUuid; + private Long lastAnalysis; + private long loc; + private String textValue; + + public String getProjectUuid() { + return projectUuid; + } + + public ProjectMeasureDto setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; + return this; + } + + public String getTextValue() { + return textValue; + } + + public ProjectMeasureDto setTextValue(String textValue) { + this.textValue = textValue; + return this; + } + + public long getLoc() { + return loc; + } + + public ProjectMeasureDto setLoc(long loc) { + this.loc = loc; + return this; + } + + public Long getLastAnalysis() { + return lastAnalysis; + } + + public ProjectMeasureDto setLastAnalysis(Long lastAnalysis) { + this.lastAnalysis = lastAnalysis; + return this; + } + +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java index 64dbe688486..ca78366e668 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java @@ -118,4 +118,7 @@ public class ProjectDao implements Dao { return session.getMapper(ProjectMapper.class); } + public List<String> selectAllProjectUuids(DbSession session) { + return mapper(session).selectAllProjectUuids(); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java index 4d8d94bc6d4..e2715aafa4f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java @@ -59,4 +59,6 @@ public interface ProjectMapper { List<ProjectDto> selectAllApplications(); List<ProjectDto> selectApplicationsByKeys(@Param("kees") Collection<String> kees); + + List<String> selectAllProjectUuids(); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java index c43e3b1dce3..082c34a4509 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java @@ -102,6 +102,10 @@ public class UserDao implements Dao { return mapper(dbSession).selectUsers(query); } + public List<UserTelemetryDto> selectUsersForTelemetry(DbSession dbSession) { + return mapper(dbSession).selectUsersForTelemetry(); + } + public UserDto insert(DbSession session, UserDto dto) { long now = system2.now(); mapper(session).insert(dto.setUuid(uuidFactory.create()).setCreatedAt(now).setUpdatedAt(now)); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java index 5e59ef4958f..353b1dc5785 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java @@ -48,6 +48,8 @@ public interface UserMapper { List<UserDto> selectUsers(UserQuery query); + List<UserTelemetryDto> selectUsersForTelemetry(); + List<UserDto> selectByLogins(List<String> logins); List<UserDto> selectByUuids(List<String> uuids); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserTelemetryDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserTelemetryDto.java new file mode 100644 index 00000000000..0c0a0c35539 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserTelemetryDto.java @@ -0,0 +1,70 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.user; + +import javax.annotation.Nullable; + +public class UserTelemetryDto { + + private String uuid; + private boolean active = true; + @Nullable + private Long lastConnectionDate; + @Nullable + private Long lastSonarlintConnectionDate; + + public String getUuid() { + return uuid; + } + + public UserTelemetryDto setUuid(String uuid) { + this.uuid = uuid; + return this; + } + + public boolean isActive() { + return active; + } + + public UserTelemetryDto setActive(boolean active) { + this.active = active; + return this; + } + + @Nullable + public Long getLastConnectionDate() { + return lastConnectionDate; + } + + public UserTelemetryDto setLastConnectionDate(@Nullable Long lastConnectionDate) { + this.lastConnectionDate = lastConnectionDate; + return this; + } + + @Nullable + public Long getLastSonarlintConnectionDate() { + return lastSonarlintConnectionDate; + } + + public UserTelemetryDto setLastSonarlintConnectionDate(@Nullable Long lastSonarlintConnectionDate) { + this.lastSonarlintConnectionDate = lastSonarlintConnectionDate; + return this; + } +} diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/alm/setting/ProjectAlmSettingMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/alm/setting/ProjectAlmSettingMapper.xml index 1b689a3c475..00bc5cc5668 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/alm/setting/ProjectAlmSettingMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/alm/setting/ProjectAlmSettingMapper.xml @@ -15,6 +15,17 @@ p.updated_at as updatedAt </sql> + <select id="selectAlmTypeAndUrlByProject" parameterType="string" resultType="org.sonar.db.alm.setting.ProjectAlmKeyAndProject"> + select + pas.project_uuid as "projectUuid", + alm_settings.alm_id as "almId", + alm_settings.url as "url" + from + project_alm_settings pas + inner join + alm_settings alm_settings on pas.alm_setting_uuid = alm_settings.uuid + </select> + <select id="selectByProjectUuid" parameterType="string" resultType="org.sonar.db.alm.setting.ProjectAlmSettingDto"> select <include refid="sqlColumns"/> from diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/AnalysisPropertiesMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/AnalysisPropertiesMapper.xml index 846ae7a2338..90f98977826 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/AnalysisPropertiesMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/AnalysisPropertiesMapper.xml @@ -31,16 +31,15 @@ AND analysis_uuid in <foreach collection="analysisUuids" open="(" close=")" item="uuid" separator=",">#{uuid,jdbcType=VARCHAR}</foreach> </select> - <select id="selectProjectCountPerAnalysisPropertyValueInLastAnalysis" parameterType="string" resultType="ProjectCountPerAnalysisPropertyValue"> + <select id="selectAnalysisPropertyValueInLastAnalysisPerProject" parameterType="string" resultType="AnalysisPropertyValuePerProject"> select - ap.text_value as "propertyValue", - count(ap.text_value) as "count" + cp.uuid as "projectUuid", ap.text_value as "propertyValue" from components cp inner join snapshots s on s.component_uuid = cp.uuid inner join analysis_properties ap on ap.analysis_uuid = s.uuid where cp.main_branch_project_uuid is null and s.islast = ${_true} and ap.kee = #{analysisPropertyKey, jdbcType=VARCHAR} - group by ap.text_value + order by cp.uuid asc </select> <insert id="insertAsEmpty" parameterType="map" useGeneratedKeys="false"> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml index 3cecf8644f9..1a4f660db53 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml @@ -111,6 +111,14 @@ pb.project_uuid = #{projectUuid, jdbcType=VARCHAR} </select> + <select id="countPrAndBranchByProjectUuid" resultType="org.sonar.db.component.PrAndBranchCountByProjectDto"> + select project_uuid as projectUuid, + sum(case when pb.branch_type = 'PULL_REQUEST' then 1 else 0 end) as pullRequest, + sum(case when pb.branch_type = 'BRANCH' then 1 else 0 end) as branch + from project_branches pb + group by pb.project_uuid + </select> + <select id="selectByUuids" resultType="org.sonar.db.component.BranchDto"> select <include refid="columns"/> from project_branches pb diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml index 4aec1f108ad..8d3c6d8d290 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml @@ -27,6 +27,47 @@ s.islast= ${_true} </select> + + <select id="selectLastMeasureForAllProjects" parameterType="map" resultType="ProjectMeasure"> + select tie_breaker.projectUuid as projectUuid, + s.build_date as lastAnalysis, + ncloc as loc, + pm.text_value as textValue + from + (select counter.projectUuid as projectUuid, + counter.maxncloc ncloc, + max(case + when c.main_branch_project_uuid is null + then 1 + else 0 + end) as mainBranch, + min(c.kee) as component_kee, + counter.projectName, + counter.projectKey + from + (select b.project_uuid as projectUuid, + p.name as projectName, + p.kee as projectKey, + max(lm.value) as maxncloc + from live_measures lm + inner join metrics m on m.uuid = lm.metric_uuid + inner join project_branches b on b.uuid = lm.component_uuid + inner join projects p on p.uuid = b.project_uuid and p.qualifier = 'TRK' + where m.name = 'ncloc' + group by b.project_uuid, p.name, p.kee) counter + inner join live_measures lmo on lmo.value = counter.maxncloc + inner join project_branches br on br.project_uuid = counter.projectUuid and br.uuid = lmo.component_uuid + inner join components c on c.uuid = br.uuid + group by counter.projectUuid, counter.maxncloc, counter.projectName, counter.projectKey) tie_breaker + inner join components c2 on c2.kee = tie_breaker.component_kee + inner join project_branches pb on c2.uuid = pb.uuid + inner join project_measures pm on pb.uuid = pm.component_uuid + inner join snapshots s on s.component_uuid = pb.uuid and s.islast = ${_true} + inner join metrics m on m.name = #{metricKey,jdbcType=VARCHAR} and m.uuid = pm.metric_uuid + where pm.analysis_uuid = s.uuid + order by ncloc desc + </select> + <select id="selectMeasure" parameterType="map" resultType="Measure"> select <include refid="measureColumns"/> from project_measures pm diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml index 5f83e3d776e..c8dd6a244a3 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml @@ -14,6 +14,12 @@ p.updated_at as updatedAt </sql> + <select id="selectAllProjectUuids" resultType="String"> + SELECT + p.uuid as uuid + FROM projects p + </select> + <select id="selectByUuid" parameterType="String" resultType="Project"> SELECT <include refid="projectColumns"/> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml index 3dfd081b18f..7265b8dde2e 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml @@ -104,6 +104,16 @@ ORDER BY u.name </select> + <select id="selectUsersForTelemetry" parameterType="map" resultType="UserTelemetry"> + SELECT + u.uuid as uuid, + u.active as "active", + u.last_connection_date as "lastConnectionDate", + u.last_sonarlint_connection as "lastSonarlintConnectionDate" + FROM users u + ORDER BY u.uuid + </select> + <select id="selectByEmail" parameterType="String" resultType="User"> SELECT <include refid="userColumns"/> diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/alm/setting/ProjectAlmSettingDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/alm/setting/ProjectAlmSettingDaoTest.java index f0a139427e2..6cc1b3968e5 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/alm/setting/ProjectAlmSettingDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/alm/setting/ProjectAlmSettingDaoTest.java @@ -130,6 +130,18 @@ public class ProjectAlmSettingDaoTest { } @Test + public void select_alm_type_and_url_by_project() { + when(uuidFactory.create()).thenReturn(A_UUID); + AlmSettingDto almSettingsDto = db.almSettings().insertGitHubAlmSetting(); + ProjectDto project = db.components().insertPrivateProjectDto(); + ProjectAlmSettingDto githubProjectAlmSettingDto = newGithubProjectAlmSettingDto(almSettingsDto, project); + underTest.insertOrUpdate(dbSession, githubProjectAlmSettingDto, almSettingsDto.getKey(), project.getName(), project.getKey()); + assertThat(underTest.selectAlmTypeAndUrlByProject(dbSession)) + .extracting(ProjectAlmKeyAndProject::getProjectUuid, ProjectAlmKeyAndProject::getAlmId, ProjectAlmKeyAndProject::getUrl) + .containsExactly(tuple(project.getUuid(), almSettingsDto.getAlm().getId(), almSettingsDto.getUrl())); + } + + @Test public void update_existing_binding() { when(uuidFactory.create()).thenReturn(A_UUID); AlmSettingDto githubAlmSetting = db.almSettings().insertGitHubAlmSetting(); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/AnalysisPropertiesDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/AnalysisPropertiesDaoTest.java index 852291698aa..23a0fc8e448 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/AnalysisPropertiesDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/AnalysisPropertiesDaoTest.java @@ -182,7 +182,7 @@ public class AnalysisPropertiesDaoTest { final String analysisPropertyKey = "key"; for (int i = 0; i < 7; i++) { final int index = i; - ProjectDto project = dbTester.components().insertPrivateProjectDto(); + ProjectDto project = dbTester.components().insertPrivateProjectDto(p -> p.setUuid("uuid" + index)); dbTester.components().insertSnapshot(project, s -> s.setLast(true).setUuid("uuid" + index)); // branches shouldn't be taken into account dbTester.components().insertProjectBranch(project); @@ -195,14 +195,18 @@ public class AnalysisPropertiesDaoTest { underTest.insert(dbSession, new AnalysisPropertyDto().setKey(analysisPropertyKey).setValue("git").setAnalysisUuid("uuid4").setUuid("4")); underTest.insert(dbSession, new AnalysisPropertyDto().setKey(analysisPropertyKey).setValue("git").setAnalysisUuid("uuid5").setUuid("5")); - List<ProjectCountPerAnalysisPropertyValue> result = underTest.selectProjectCountPerAnalysisPropertyValueInLastAnalysis(dbSession, analysisPropertyKey); + List<AnalysisPropertyValuePerProject> result = underTest.selectAnalysisPropertyValueInLastAnalysisPerProject(dbSession, analysisPropertyKey); assertThat(result) - .extracting(ProjectCountPerAnalysisPropertyValue::getPropertyValue, ProjectCountPerAnalysisPropertyValue::getCount) + .extracting(AnalysisPropertyValuePerProject::getProjectUuid, AnalysisPropertyValuePerProject::getPropertyValue) .containsExactlyInAnyOrder( - tuple("git", 3L), - tuple("svn", 1L), - tuple("undetected", 2L)); + tuple("uuid0", "git"), + tuple("uuid1", "svn"), + tuple("uuid2", "undetected"), + tuple("uuid3", "undetected"), + tuple("uuid4", "git"), + tuple("uuid5", "git") + ); } private AnalysisPropertyDto insertAnalysisPropertyDto(int valueLength) { diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java index 473fa4c23f2..f9439593db1 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java @@ -49,6 +49,8 @@ import static org.apache.commons.lang.StringUtils.repeat; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.assertj.core.api.Assertions.tuple; +import static org.sonar.db.component.BranchType.BRANCH; +import static org.sonar.db.component.BranchType.PULL_REQUEST; @RunWith(DataProviderRunner.class) public class BranchDaoTest { @@ -546,6 +548,30 @@ public class BranchDaoTest { } @Test + public void countPrAndBranchByProjectUuid() { + ComponentDto project1 = db.components().insertPrivateProject(); + db.components().insertProjectBranch(project1, b -> b.setBranchType(BRANCH).setKey("p1-branch-1")); + db.components().insertProjectBranch(project1, b -> b.setBranchType(BRANCH).setKey("p1-branch-2")); + db.components().insertProjectBranch(project1, b -> b.setBranchType(PULL_REQUEST).setKey("p1-pr-1")); + db.components().insertProjectBranch(project1, b -> b.setBranchType(PULL_REQUEST).setKey("p1-pr-2")); + db.components().insertProjectBranch(project1, b -> b.setBranchType(PULL_REQUEST).setKey("p1-pr-3")); + + ComponentDto project2 = db.components().insertPrivateProject(); + db.components().insertProjectBranch(project2, b -> b.setBranchType(PULL_REQUEST).setKey("p2-pr-1")); + + ComponentDto project3 = db.components().insertPrivateProject(); + db.components().insertProjectBranch(project3, b -> b.setBranchType(BRANCH).setKey("p3-branch-1")); + + assertThat(underTest.countPrAndBranchByProjectUuid(db.getSession())) + .extracting(PrAndBranchCountByProjectDto::getProjectUuid, PrAndBranchCountByProjectDto::getBranch, PrAndBranchCountByProjectDto::getPullRequest) + .containsExactlyInAnyOrder( + tuple(project1.uuid(), 3L, 3L), + tuple(project2.uuid(), 1L, 1L), + tuple(project3.uuid(), 2L, 0L) + ); + } + + @Test public void selectProjectUuidsWithIssuesNeedSync() { ComponentDto project1 = db.components().insertPrivateProject(); ComponentDto project2 = db.components().insertPrivateProject(); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java index 6d2005ef246..bbde6fa6905 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java @@ -575,9 +575,39 @@ public class UserDaoTest { .containsExactlyInAnyOrder(tuple(u1.getLogin(), u1.getName()), tuple(u2.getLogin(), u2.getName())); } - private void commit(Runnable runnable) { - runnable.run(); - session.commit(); + @Test + public void selectUserTelemetry() { + UserDto u1 = insertUser(true); + UserDto u2 = insertUser(false); + + List<UserTelemetryDto> result = underTest.selectUsersForTelemetry(db.getSession()); + + assertThat(result) + .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::isActive, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate) + .containsExactlyInAnyOrder( + tuple(u1.getUuid(), u1.isActive(), u1.getLastConnectionDate(), u1.getLastSonarlintConnectionDate()), + tuple(u2.getUuid(), u2.isActive(), u2.getLastConnectionDate(), u2.getLastSonarlintConnectionDate())); + } + + @Test + public void selectUserTelemetryUpdatedLastConnectionDate() { + UserDto u1 = insertUser(true); + UserDto u2 = insertUser(false); + + assertThat(underTest.selectUsersForTelemetry(db.getSession())) + .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::isActive, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate) + .containsExactlyInAnyOrder( + tuple(u1.getUuid(), u1.isActive(), null, u1.getLastSonarlintConnectionDate()), + tuple(u2.getUuid(), u2.isActive(), null, u2.getLastSonarlintConnectionDate())); + + underTest.update(db.getSession(), u1.setLastConnectionDate(10_000_000_000L)); + underTest.update(db.getSession(), u2.setLastConnectionDate(20_000_000_000L)); + + assertThat(underTest.selectUsersForTelemetry(db.getSession())) + .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::isActive, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate) + .containsExactlyInAnyOrder( + tuple(u1.getUuid(), u1.isActive(), 10_000_000_000L, u1.getLastSonarlintConnectionDate()), + tuple(u2.getUuid(), u2.isActive(), 20_000_000_000L, u2.getLastSonarlintConnectionDate())); } private UserDto insertActiveUser() { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java index 89d5f724130..0f54264d80b 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java @@ -22,9 +22,11 @@ package org.sonar.server.telemetry; import java.util.List; import java.util.Map; import java.util.Optional; +import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.core.platform.EditionProvider; import org.sonar.core.platform.EditionProvider.Edition; +import org.sonar.db.user.UserTelemetryDto; import org.sonar.server.measure.index.ProjectMeasuresStatistics; import static java.util.Collections.emptyList; @@ -40,11 +42,8 @@ public class TelemetryData { private final boolean usingBranches; private final Database database; private final Map<String, Long> projectCountByLanguage; - private final Map<String, Long> almIntegrationCountByAlm; private final Map<String, Long> nclocByLanguage; private final List<String> externalAuthenticationProviders; - private final Map<String, Long> projectCountByScm; - private final Map<String, Long> projectCountByCi; private final EditionProvider.Edition edition; private final String licenseType; private final Long installationDate; @@ -55,6 +54,9 @@ public class TelemetryData { private final List<String> customSecurityConfigs; private final long sonarlintWeeklyUsers; private final long numberOfConnectedSonarLintClients; + private final List<UserTelemetryDto> users; + private final List<Project> projects; + private final List<ProjectStatistics> projectStatistics; private TelemetryData(Builder builder) { serverId = builder.serverId; @@ -67,7 +69,6 @@ public class TelemetryData { database = builder.database; sonarlintWeeklyUsers = builder.sonarlintWeeklyUsers; projectCountByLanguage = builder.projectMeasuresStatistics.getProjectCountByLanguage(); - almIntegrationCountByAlm = builder.almIntegrationCountByAlm; nclocByLanguage = builder.projectMeasuresStatistics.getNclocByLanguage(); edition = builder.edition; licenseType = builder.licenseType; @@ -78,9 +79,10 @@ public class TelemetryData { hasUnanalyzedCpp = builder.hasUnanalyzedCpp; customSecurityConfigs = builder.customSecurityConfigs == null ? emptyList() : builder.customSecurityConfigs; externalAuthenticationProviders = builder.externalAuthenticationProviders; - projectCountByScm = builder.projectCountByScm; - projectCountByCi = builder.projectCountByCi; numberOfConnectedSonarLintClients = builder.numberOfConnectedSonarLintClients; + users = builder.users; + projects = builder.projects; + projectStatistics = builder.projectStatistics; } public String getServerId() { @@ -127,10 +129,6 @@ public class TelemetryData { return projectCountByLanguage; } - public Map<String, Long> getAlmIntegrationCountByAlm() { - return almIntegrationCountByAlm; - } - public Map<String, Long> getNclocByLanguage() { return nclocByLanguage; } @@ -171,12 +169,16 @@ public class TelemetryData { return externalAuthenticationProviders; } - public Map<String, Long> getProjectCountByScm() { - return projectCountByScm; + public List<UserTelemetryDto> getUserTelemetries() { + return users; + } + + public List<Project> getProjects() { + return projects; } - public Map<String, Long> getProjectCountByCi() { - return projectCountByCi; + public List<ProjectStatistics> getProjectStatistics() { + return projectStatistics; } static Builder builder() { @@ -191,7 +193,6 @@ public class TelemetryData { private Map<String, String> plugins; private Database database; private ProjectMeasuresStatistics projectMeasuresStatistics; - private Map<String, Long> almIntegrationCountByAlm; private Long ncloc; private Boolean usingBranches; private Edition edition; @@ -203,9 +204,10 @@ public class TelemetryData { private Boolean hasUnanalyzedCpp; private List<String> customSecurityConfigs; private List<String> externalAuthenticationProviders; - private Map<String, Long> projectCountByScm; - private Map<String, Long> projectCountByCi; private long numberOfConnectedSonarLintClients; + private List<UserTelemetryDto> users; + private List<Project> projects; + private List<ProjectStatistics> projectStatistics; private Builder() { // enforce static factory method @@ -216,21 +218,11 @@ public class TelemetryData { return this; } - Builder setProjectCountByScm(Map<String, Long> projectCountByScm) { - this.projectCountByScm = projectCountByScm; - return this; - } - Builder setSonarlintWeeklyUsers(long sonarlintWeeklyUsers) { this.sonarlintWeeklyUsers = sonarlintWeeklyUsers; return this; } - Builder setProjectCountByCi(Map<String, Long> projectCountByCi) { - this.projectCountByCi = projectCountByCi; - return this; - } - Builder setServerId(String serverId) { this.serverId = serverId; return this; @@ -251,11 +243,6 @@ public class TelemetryData { return this; } - Builder setAlmIntegrationCountByAlm(Map<String, Long> almIntegrationCountByAlm) { - this.almIntegrationCountByAlm = almIntegrationCountByAlm; - return this; - } - Builder setProjectMeasuresStatistics(ProjectMeasuresStatistics projectMeasuresStatistics) { this.projectMeasuresStatistics = projectMeasuresStatistics; return this; @@ -321,21 +308,33 @@ public class TelemetryData { return this; } + Builder setUsers(List<UserTelemetryDto> users) { + this.users = users; + return this; + } + + Builder setProjects(List<Project> projects) { + this.projects = projects; + return this; + } + TelemetryData build() { requireNonNull(serverId); requireNonNull(version); requireNonNull(plugins); requireNonNull(projectMeasuresStatistics); - requireNonNull(almIntegrationCountByAlm); requireNonNull(ncloc); requireNonNull(database); requireNonNull(usingBranches); requireNonNull(externalAuthenticationProviders); - requireNonNull(projectCountByScm); - requireNonNull(projectCountByCi); return new TelemetryData(this); } + + Builder setProjectStatistics(List<ProjectStatistics> projectStatistics) { + this.projectStatistics = projectStatistics; + return this; + } } static class Database { @@ -355,4 +354,79 @@ public class TelemetryData { return version; } } + + static class Project { + private final String projectUuid; + private final String language; + private final Long loc; + private final Long lastAnalysis; + + public Project(String projectUuid, Long lastAnalysis, String language, Long loc) { + this.projectUuid = projectUuid; + this.lastAnalysis = lastAnalysis; + this.language = language; + this.loc = loc; + } + + public String getProjectUuid() { + return projectUuid; + } + + public String getLanguage() { + return language; + } + + public Long getLoc() { + return loc; + } + + public Long getLastAnalysis() { + return lastAnalysis; + } + } + + static class ProjectStatistics{ + private final String projectUuid; + private final Long branchCount; + private final Long pullRequestCount; + private final String scm; + private final String ci; + private final String alm; + + ProjectStatistics(String projectUuid, Long branchCount, Long pullRequestCount, @Nullable String scm, @Nullable String ci, @Nullable String alm) { + this.projectUuid = projectUuid; + this.branchCount = branchCount; + this.pullRequestCount = pullRequestCount; + this.scm = scm; + this.ci = ci; + this.alm = alm; + } + + public String getProjectUuid() { + return projectUuid; + } + + public Long getBranchCount() { + return branchCount; + } + + public Long getPullRequestCount() { + return pullRequestCount; + } + + @CheckForNull + public String getScm() { + return scm; + } + + @CheckForNull + public String getCi() { + return ci; + } + + @CheckForNull + public String getAlm() { + return alm; + } + } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java index a58658519af..e6d0c71885a 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java @@ -19,14 +19,20 @@ */ package org.sonar.server.telemetry; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Locale; +import org.jetbrains.annotations.NotNull; import org.sonar.api.utils.text.JsonWriter; import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; +import static org.sonar.api.utils.DateUtils.DATETIME_FORMAT; public class TelemetryDataJsonWriter { - public static final String COUNT = "count"; + public static final String COUNT_PROP = "count"; + public static final String LANGUAGE_PROP = "language"; public void writeTelemetryData(JsonWriter json, TelemetryData statistics) { json.beginObject(); @@ -56,8 +62,8 @@ public class TelemetryDataJsonWriter { json.beginArray(); statistics.getProjectCountByLanguage().forEach((language, count) -> { json.beginObject(); - json.prop("language", language); - json.prop(COUNT, count); + json.prop(LANGUAGE_PROP, language); + json.prop(COUNT_PROP, count); json.endObject(); }); json.endArray(); @@ -65,20 +71,11 @@ public class TelemetryDataJsonWriter { json.beginArray(); statistics.getNclocByLanguage().forEach((language, ncloc) -> { json.beginObject(); - json.prop("language", language); + json.prop(LANGUAGE_PROP, language); json.prop("ncloc", ncloc); json.endObject(); }); json.endArray(); - json.name("almIntegrationCount"); - json.beginArray(); - statistics.getAlmIntegrationCountByAlm().forEach((alm, count) -> { - json.beginObject(); - json.prop("alm", alm); - json.prop(COUNT, count); - json.endObject(); - }); - json.endArray(); if (!statistics.getCustomSecurityConfigs().isEmpty()) { json.name("customSecurityConfig"); @@ -95,9 +92,6 @@ public class TelemetryDataJsonWriter { statistics.getExternalAuthenticationProviders().forEach(json::value); json.endArray(); - addScmInfo(json, statistics); - addCiInfo(json, statistics); - json.prop("sonarlintWeeklyUsers", statistics.sonarlintWeeklyUsers()); if (statistics.getInstallationDate() != null) { @@ -107,30 +101,77 @@ public class TelemetryDataJsonWriter { json.prop("installationVersion", statistics.getInstallationVersion()); } json.prop("docker", statistics.isInDocker()); + + writeUserData(json, statistics); + writeProjectData(json, statistics); + writeProjectStatsData(json, statistics); + json.endObject(); } - private static void addScmInfo(JsonWriter json, TelemetryData statistics) { - json.name("projectCountByScm"); - json.beginArray(); - statistics.getProjectCountByScm().forEach((scm, count) -> { - json.beginObject(); - json.prop("scm", scm); - json.prop(COUNT, count); - json.endObject(); - }); - json.endArray(); + private static void writeUserData(JsonWriter json, TelemetryData statistics) { + if (statistics.getUserTelemetries() != null) { + json.name("users"); + json.beginArray(); + statistics.getUserTelemetries().forEach(user -> { + json.beginObject(); + json.prop("userUuid", user.getUuid()); + json.prop("status", user.isActive() ? "active" : "inactive"); + + if (user.getLastConnectionDate() != null) { + json.prop("lastActivity", toUtc(user.getLastConnectionDate())); + } + if (user.getLastSonarlintConnectionDate() != null) { + json.prop("lastSonarlintActivity", toUtc(user.getLastSonarlintConnectionDate())); + } + + json.endObject(); + }); + json.endArray(); + } } - private static void addCiInfo(JsonWriter json, TelemetryData statistics) { - json.name("projectCountByCI"); - json.beginArray(); - statistics.getProjectCountByCi().forEach((ci, count) -> { - json.beginObject(); - json.prop("ci", ci); - json.prop(COUNT, count); - json.endObject(); - }); - json.endArray(); + private static void writeProjectData(JsonWriter json, TelemetryData statistics) { + if (statistics.getProjects() != null) { + json.name("projects"); + json.beginArray(); + statistics.getProjects().forEach(project -> { + json.beginObject(); + json.prop("projectUuid", project.getProjectUuid()); + if (project.getLastAnalysis() != null) { + json.prop("lastAnalysis", toUtc(project.getLastAnalysis())); + } + json.prop(LANGUAGE_PROP, project.getLanguage()); + json.prop("loc", project.getLoc()); + json.endObject(); + }); + json.endArray(); + } } + + private static void writeProjectStatsData(JsonWriter json, TelemetryData statistics) { + if (statistics.getProjectStatistics() != null) { + json.name("projects-general-stats"); + json.beginArray(); + statistics.getProjectStatistics().forEach(project -> { + json.beginObject(); + json.prop("projectUuid", project.getProjectUuid()); + json.prop("branchCount", project.getBranchCount()); + json.prop("pullRequestCount", project.getPullRequestCount()); + json.prop("scm", project.getScm()); + json.prop("ci", project.getCi()); + json.prop("alm", project.getAlm()); + json.endObject(); + }); + json.endArray(); + } + } + + @NotNull + private static String toUtc(long date) { + return DateTimeFormatter.ofPattern(DATETIME_FORMAT) + .withZone(ZoneOffset.UTC) + .format(Instant.ofEpochMilli(date)); + } + } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java index aa280713ec1..d98051ee11a 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java @@ -19,22 +19,25 @@ */ package org.sonar.server.telemetry; -import com.google.common.collect.ImmutableMap; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.StringWriter; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.jetbrains.annotations.NotNull; import org.junit.Test; import org.junit.runner.RunWith; import org.sonar.api.utils.text.JsonWriter; import org.sonar.core.platform.EditionProvider; import org.sonar.core.util.stream.MoreCollectors; +import org.sonar.db.user.UserTelemetryDto; import org.sonar.server.measure.index.ProjectMeasuresStatistics; import static java.util.Arrays.asList; @@ -50,7 +53,6 @@ public class TelemetryDataJsonWriterTest { .setServerId("foo") .setVersion("bar") .setPlugins(Collections.emptyMap()) - .setAlmIntegrationCountByAlm(Collections.emptyMap()) .setProjectMeasuresStatistics(ProjectMeasuresStatistics.builder() .setProjectCount(12) .setProjectCountByLanguage(Collections.emptyMap()) @@ -58,10 +60,8 @@ public class TelemetryDataJsonWriterTest { .build()) .setNcloc(42L) .setExternalAuthenticationProviders(asList("github", "gitlab")) - .setProjectCountByScm(Collections.emptyMap()) .setSonarlintWeeklyUsers(10) .setNumberOfConnectedSonarLintClients(5) - .setProjectCountByCi(Collections.emptyMap()) .setDatabase(new TelemetryData.Database("H2", "11")) .setUsingBranches(true); @@ -230,40 +230,6 @@ public class TelemetryDataJsonWriterTest { } @Test - public void write_project_count_by_scm() { - TelemetryData data = SOME_TELEMETRY_DATA - .setProjectCountByScm(ImmutableMap.of("git", 5L, "svn", 4L, "cvs", 3L, "undetected", 2L)) - .build(); - - String json = writeTelemetryData(data); - - assertJson(json).isSimilarTo("{" + - " \"projectCountByScm\": [" - + "{ \"scm\":\"git\", \"count\":5}," - + "{ \"scm\":\"svn\", \"count\":4}," - + "{ \"scm\":\"cvs\", \"count\":3}," - + "{ \"scm\":\"undetected\", \"count\":2}," - + "]}"); - } - - @Test - public void write_project_count_by_ci() { - TelemetryData data = SOME_TELEMETRY_DATA - .setProjectCountByCi(ImmutableMap.of("Bitbucket Pipelines", 5L, "Github Actions", 4L, "Jenkins", 3L, "undetected", 2L)) - .build(); - - String json = writeTelemetryData(data); - - assertJson(json).isSimilarTo("{" + - " \"projectCountByCI\": [" - + "{ \"ci\":\"Bitbucket Pipelines\", \"count\":5}," - + "{ \"ci\":\"Github Actions\", \"count\":4}," - + "{ \"ci\":\"Jenkins\", \"count\":3}," - + "{ \"ci\":\"undetected\", \"count\":2}," - + "]}"); - } - - @Test public void write_project_stats_by_language() { int projectCount = random.nextInt(8909); Map<String, Long> countByLanguage = IntStream.range(0, 1 + random.nextInt(10)) @@ -296,32 +262,6 @@ public class TelemetryDataJsonWriterTest { } @Test - public void write_alm_count_by_alm() { - TelemetryData data = SOME_TELEMETRY_DATA - .setAlmIntegrationCountByAlm(ImmutableMap.of( - "github", 4L, - "github_cloud", 1L, - "gitlab", 2L, - "gitlab_cloud", 5L, - "azure_devops", 1L)) - .build(); - - String json = writeTelemetryData(data); - - assertJson(json).isSimilarTo("{" + - " \"almIntegrationCount\": " + - "[" - + "{ \"alm\":\"github\", \"count\":4}," - + "{ \"alm\":\"github_cloud\", \"count\":1}," - + "{ \"alm\":\"gitlab\", \"count\":2}," - + "{ \"alm\":\"gitlab_cloud\", \"count\":5}," - + "{ \"alm\":\"azure_devops\", \"count\":1}," - + "]" - + - "}"); - } - - @Test public void does_not_write_installation_date_if_null() { TelemetryData data = SOME_TELEMETRY_DATA .setInstallationDate(null) @@ -413,6 +353,122 @@ public class TelemetryDataJsonWriterTest { "}"); } + @Test + public void writes_all_users() { + TelemetryData data = SOME_TELEMETRY_DATA + .setUsers(getUsers()) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo("{" + + " \"users\": [" + + " {" + + " \"userUuid\":\"uuid-0\"," + + " \"lastActivity\":\"1970-01-01T00:00:00+0000\"," + + " \"lastSonarlintActivity\":\"1970-01-01T00:00:00+0000\"," + + " \"status\":\"active\"" + + " }," + + " {" + + " \"userUuid\":\"uuid-1\"," + + " \"lastActivity\":\"1970-01-01T00:00:00+0000\"," + + " \"lastSonarlintActivity\":\"1970-01-01T00:00:00+0000\"," + + " \"status\":\"inactive\"" + + " }," + + " {" + + " \"userUuid\":\"uuid-2\"," + + " \"lastActivity\":\"1970-01-01T01:00:00+0100\"," + + " \"lastSonarlintActivity\":\"1970-01-01T00:00:00+0000\"," + + " \"status\":\"active\"" + + " }" + + " ]" + + "}"); + } + + @Test + public void writes_all_projects() { + TelemetryData data = SOME_TELEMETRY_DATA + .setProjects(getProjects()) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo("{" + + " \"projects\": [" + + " {" + + " \"projectUuid\": \"uuid-0\"," + + " \"language\": \"lang-0\"," + + " \"lastAnalysis\":\"1970-01-01T00:00:00+0000\"," + + " \"loc\": 2" + + " }," + + " {" + + " \"projectUuid\": \"uuid-1\"," + + " \"language\": \"lang-1\"," + + " \"lastAnalysis\":\"1970-01-01T00:00:00+0000\"," + + " \"loc\": 4" + + " }," + + " {" + + " \"projectUuid\": \"uuid-2\"," + + " \"language\": \"lang-2\"," + + " \"lastAnalysis\":\"1970-01-01T00:00:00+0000\"," + + " \"loc\": 6" + + " }" + + " ]" + + "}"); + } + + @Test + public void writes_all_projects_stats() { + TelemetryData data = SOME_TELEMETRY_DATA + .setProjectStatistics(getProjectStats()) + .build(); + + String json = writeTelemetryData(data); + + assertJson(json).isSimilarTo("{" + + " \"projects-general-stats\": [" + + " {" + + " \"projectUuid\": \"uuid-0\"," + + " \"branchCount\": 2," + + " \"pullRequestCount\": 2," + + " \"scm\": \"scm-0\"," + + " \"ci\": \"ci-0\"," + + " \"alm\": \"alm-0\"" + + " }," + + " {" + + " \"projectUuid\": \"uuid-1\"," + + " \"branchCount\": 4," + + " \"pullRequestCount\": 4," + + " \"scm\": \"scm-1\"," + + " \"ci\": \"ci-1\"," + + " \"alm\": \"alm-1\"" + + " }," + + " {" + + " \"projectUuid\": \"uuid-2\"," + + " \"branchCount\": 6," + + " \"pullRequestCount\": 6," + + " \"scm\": \"scm-2\"," + + " \"ci\": \"ci-2\"," + + " \"alm\": \"alm-2\"" + + " }" + + " ]" + + "}"); + } + + @NotNull + private static List<UserTelemetryDto> getUsers() { + return IntStream.range(0, 3).mapToObj(i -> new UserTelemetryDto().setUuid("uuid-" + i).setActive(i % 2 == 0).setLastConnectionDate(1L).setLastSonarlintConnectionDate(2L)).collect(Collectors.toList()); + } + + private static List<TelemetryData.Project> getProjects() { + return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.Project("uuid-" + i, 1L, "lang-" + i, (i + 1L) * 2L)).collect(Collectors.toList()); + } + + private List<TelemetryData.ProjectStatistics> getProjectStats() { + return IntStream.range(0, 3).mapToObj(i -> new TelemetryData.ProjectStatistics("uuid-" + i, (i + 1L) * 2L, (i + 1L) * 2L, "scm-" + i, "ci-" + i, "alm-" + i)) + .collect(Collectors.toList()); + } + @DataProvider public static Object[][] allEditions() { return Arrays.stream(EditionProvider.Edition.values()) diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java index 23544746705..1a5c0296c75 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDaemon.java @@ -22,7 +22,6 @@ package org.sonar.server.telemetry; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.io.StringWriter; -import java.util.Date; import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -38,8 +37,6 @@ import org.sonar.api.utils.text.JsonWriter; import org.sonar.server.property.InternalProperties; import org.sonar.server.util.GlobalLockManager; -import static org.sonar.api.utils.DateUtils.formatDate; -import static org.sonar.api.utils.DateUtils.parseDate; import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_ENABLE; import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_FREQUENCY_IN_SECONDS; import static org.sonar.process.ProcessProperties.Property.SONAR_TELEMETRY_URL; diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java index 8caa743ca16..75d162fcdc5 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java @@ -21,6 +21,7 @@ package org.sonar.server.telemetry; import java.sql.DatabaseMetaData; import java.sql.SQLException; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -33,7 +34,6 @@ import javax.inject.Inject; import org.sonar.api.config.Configuration; import org.sonar.api.platform.Server; import org.sonar.api.server.ServerSide; -import org.sonar.core.config.CorePropertyDefinitions; import org.sonar.core.platform.PlatformEditionProvider; import org.sonar.core.platform.PluginInfo; import org.sonar.core.platform.PluginRepository; @@ -41,8 +41,10 @@ import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.alm.setting.ALM; -import org.sonar.db.alm.setting.AlmSettingDto; -import org.sonar.db.component.ProjectCountPerAnalysisPropertyValue; +import org.sonar.db.alm.setting.ProjectAlmKeyAndProject; +import org.sonar.db.component.AnalysisPropertyValuePerProject; +import org.sonar.db.component.PrAndBranchCountByProjectDto; +import org.sonar.db.measure.ProjectMeasureDto; import org.sonar.db.measure.SumNclocDbQuery; import org.sonar.server.es.SearchOptions; import org.sonar.server.measure.index.ProjectMeasuresIndex; @@ -55,7 +57,10 @@ import org.sonar.server.user.index.UserQuery; import static java.util.Arrays.asList; import static java.util.Optional.ofNullable; -import static org.apache.commons.lang.StringUtils.startsWith; +import static org.sonar.api.internal.apachecommons.lang.StringUtils.startsWithIgnoreCase; +import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM; import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; import static org.sonar.core.platform.EditionProvider.Edition.DATACENTER; import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; @@ -64,6 +69,7 @@ import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY @ServerSide public class TelemetryDataLoaderImpl implements TelemetryDataLoader { + public static final String UNDETECTED = "undetected"; private final Server server; private final DbClient dbClient; private final PluginRepository pluginRepository; @@ -134,11 +140,47 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { data.setHasUnanalyzedCpp(numberOfUnanalyzedCppMeasures > 0); }); - data.setAlmIntegrationCountByAlm(countAlmUsage(dbSession)); data.setExternalAuthenticationProviders(dbClient.userDao().selectExternalIdentityProviders(dbSession)); data.setSonarlintWeeklyUsers(dbClient.userDao().countSonarlintWeeklyUsers(dbSession)); - addScmInformationToTelemetry(dbSession, data); - addCiInformationToTelemetry(dbSession, data); + + Map<String, String> scmByProject = getAnalysisPropertyByProject(dbSession, SONAR_ANALYSIS_DETECTEDSCM); + Map<String, String> ciByProject = getAnalysisPropertyByProject(dbSession, SONAR_ANALYSIS_DETECTEDCI); + Map<String, ProjectAlmKeyAndProject> almAndUrlByProject = getAlmAndUrlByProject(dbSession); + List<String> projectUuids = dbClient.projectDao().selectAllProjectUuids(dbSession); + + Map<String, PrAndBranchCountByProjectDto> prAndBranchCountByProjects = dbClient.branchDao().countPrAndBranchByProjectUuid(dbSession) + .stream().collect(Collectors.toMap(PrAndBranchCountByProjectDto::getProjectUuid, Function.identity())); + + List<TelemetryData.ProjectStatistics> projectStatistics = new ArrayList<>(); + for (String projectUuid : projectUuids) { + Long branchCount = Optional.ofNullable(prAndBranchCountByProjects.get(projectUuid)).map(PrAndBranchCountByProjectDto::getBranch).orElse(0L); + Long pullRequestCount = Optional.ofNullable(prAndBranchCountByProjects.get(projectUuid)).map(PrAndBranchCountByProjectDto::getPullRequest).orElse(0L); + String scm = Optional.ofNullable(scmByProject.get(projectUuid)).orElse(UNDETECTED); + String ci = Optional.ofNullable(ciByProject.get(projectUuid)).orElse(UNDETECTED); + String alm = null; + if (almAndUrlByProject.containsKey(projectUuid)) { + ProjectAlmKeyAndProject projectAlmKeyAndProject = almAndUrlByProject.get(projectUuid); + alm = getAlmName(projectAlmKeyAndProject.getAlmId(), projectAlmKeyAndProject.getUrl()); + } + alm = Optional.ofNullable(alm).orElse(UNDETECTED); + + projectStatistics.add(new TelemetryData.ProjectStatistics(projectUuid, branchCount, pullRequestCount, scm, ci, alm)); + } + data.setProjectStatistics(projectStatistics); + + data.setUsers(dbClient.userDao().selectUsersForTelemetry(dbSession)); + + List<ProjectMeasureDto> measures = dbClient.measureDao().selectLastMeasureForAllProjects(dbSession, NCLOC_LANGUAGE_DISTRIBUTION_KEY); + List<TelemetryData.Project> projects = new ArrayList<>(); + for (ProjectMeasureDto measure : measures) { + for (String measureTextValue : measure.getTextValue().split(";")) { + String[] languageAndLoc = measureTextValue.split("="); + String language = languageAndLoc[0]; + Long loc = Long.parseLong(languageAndLoc[1]); + projects.add(new TelemetryData.Project(measure.getProjectUuid(), measure.getLastAnalysis(), language, loc)); + } + } + data.setProjects(projects); } setSecurityCustomConfigIfPresent(data); @@ -168,40 +210,33 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { }); } - private void addScmInformationToTelemetry(DbSession dbSession, TelemetryData.Builder data) { - Map<String, Long> projectCountPerScmDetected = dbClient.analysisPropertiesDao() - .selectProjectCountPerAnalysisPropertyValueInLastAnalysis(dbSession, CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM) + private Map<String, String> getAnalysisPropertyByProject(DbSession dbSession, String analysisPropertyKey) { + return dbClient.analysisPropertiesDao() + .selectAnalysisPropertyValueInLastAnalysisPerProject(dbSession, analysisPropertyKey) .stream() - .collect(Collectors.toMap(ProjectCountPerAnalysisPropertyValue::getPropertyValue, ProjectCountPerAnalysisPropertyValue::getCount)); - data.setProjectCountByScm(projectCountPerScmDetected); + .collect(Collectors.toMap(AnalysisPropertyValuePerProject::getProjectUuid, AnalysisPropertyValuePerProject::getPropertyValue)); } - private void addCiInformationToTelemetry(DbSession dbSession, TelemetryData.Builder data) { - Map<String, Long> projectCountPerCiDetected = dbClient.analysisPropertiesDao() - .selectProjectCountPerAnalysisPropertyValueInLastAnalysis(dbSession, CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI) - .stream() - .collect(Collectors.toMap(ProjectCountPerAnalysisPropertyValue::getPropertyValue, ProjectCountPerAnalysisPropertyValue::getCount)); - data.setProjectCountByCi(projectCountPerCiDetected); + private Map<String, ProjectAlmKeyAndProject> getAlmAndUrlByProject(DbSession dbSession) { + List<ProjectAlmKeyAndProject> projectAlmKeyAndProjects = dbClient.projectAlmSettingDao().selectAlmTypeAndUrlByProject(dbSession); + return projectAlmKeyAndProjects.stream().collect(Collectors.toMap(ProjectAlmKeyAndProject::getProjectUuid, Function.identity())); } - private Map<String, Long> countAlmUsage(DbSession dbSession) { - return dbClient.almSettingDao().selectAll(dbSession).stream() - .collect(Collectors.groupingBy(almSettingDto -> { - if (checkIfCloudAlm(almSettingDto, ALM.GITHUB, "https://api.github.com")) { - return "github_cloud"; - } else if (checkIfCloudAlm(almSettingDto, ALM.GITLAB, "https://gitlab.com/api/v4")) { - return "gitlab_cloud"; - } else if (checkIfCloudAlm(almSettingDto, ALM.AZURE_DEVOPS, "https://dev.azure.com")) { - return "azure_devops_cloud"; - } else if (ALM.BITBUCKET_CLOUD.equals(almSettingDto.getAlm())) { - return almSettingDto.getRawAlm(); - } - return almSettingDto.getRawAlm() + "_server"; - }, Collectors.counting())); + private static String getAlmName(String alm, String url) { + if (checkIfCloudAlm(alm, ALM.GITHUB.getId(), url, "https://api.github.com")) { + return "github_cloud"; + } else if (checkIfCloudAlm(alm, ALM.GITLAB.getId(), url, "https://gitlab.com/api/v4")) { + return "gitlab_cloud"; + } else if (checkIfCloudAlm(alm, ALM.AZURE_DEVOPS.getId(), url, "https://dev.azure.com")) { + return "azure_devops_cloud"; + } else if (ALM.BITBUCKET_CLOUD.getId().equals(alm)) { + return alm; + } + return alm + "_server"; } - private static boolean checkIfCloudAlm(AlmSettingDto almSettingDto, ALM alm, String url) { - return alm.equals(almSettingDto.getAlm()) && startsWith(almSettingDto.getUrl(), url); + private static boolean checkIfCloudAlm(String almRaw, String alm, String url, String cloudUrl) { + return alm.equals(almRaw) && startsWithIgnoreCase(url, cloudUrl); } @Override diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java index 1a4f4b48d82..155a3598d0f 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDaemonTest.java @@ -65,10 +65,7 @@ public class TelemetryDaemonTest { .setServerId("foo") .setVersion("bar") .setPlugins(Collections.emptyMap()) - .setAlmIntegrationCountByAlm(Collections.emptyMap()) .setExternalAuthenticationProviders(singletonList("github")) - .setProjectCountByScm(Collections.emptyMap()) - .setProjectCountByCi(Collections.emptyMap()) .setProjectMeasuresStatistics(ProjectMeasuresStatistics.builder() .setProjectCount(12) .setProjectCountByLanguage(Collections.emptyMap()) diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java index 1d3fac27fe6..590bcef4b18 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/telemetry/TelemetryDataLoaderImplTest.java @@ -23,6 +23,7 @@ import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.IntStream; import org.junit.Rule; import org.junit.Test; @@ -33,8 +34,13 @@ import org.sonar.core.platform.PluginInfo; import org.sonar.core.platform.PluginRepository; import org.sonar.db.DbSession; import org.sonar.db.DbTester; +import org.sonar.db.alm.setting.AlmSettingDto; +import org.sonar.db.component.AnalysisPropertyDto; import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.SnapshotDto; import org.sonar.db.metric.MetricDto; +import org.sonar.db.user.UserDto; +import org.sonar.db.user.UserTelemetryDto; import org.sonar.server.es.EsTester; import org.sonar.server.measure.index.ProjectMeasuresIndex; import org.sonar.server.measure.index.ProjectMeasuresIndexer; @@ -50,6 +56,7 @@ import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; +import static org.assertj.core.groups.Tuple.tuple; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -57,6 +64,8 @@ import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY; import static org.sonar.api.measures.CoreMetrics.LINES_KEY; import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; import static org.sonar.api.measures.CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDCI; +import static org.sonar.core.config.CorePropertyDefinitions.SONAR_ANALYSIS_DETECTEDSCM; import static org.sonar.core.platform.EditionProvider.Edition.COMMUNITY; import static org.sonar.core.platform.EditionProvider.Edition.DEVELOPER; import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE; @@ -93,15 +102,24 @@ public class TelemetryDataLoaderImplTest { public void send_telemetry_data() { String serverId = "AU-TpxcB-iU5OvuD2FL7"; String version = "7.5.4"; + Long analysisDate = 1L; + Long lastConnectionDate = 5L; + server.setId(serverId); server.setVersion(version); List<PluginInfo> plugins = asList(newPlugin("java", "4.12.0.11033"), newPlugin("scmgit", "1.2"), new PluginInfo("other")); when(pluginRepository.getPluginInfos()).thenReturn(plugins); when(editionProvider.get()).thenReturn(Optional.of(DEVELOPER)); - int userCount = 3; - IntStream.range(0, userCount).forEach(i -> db.users().insertUser(u -> u.setExternalIdentityProvider("provider" + i))); - db.users().insertUser(u -> u.setActive(false).setExternalIdentityProvider("provider0")); + int activeUserCount = 3; + List<UserDto> activeUsers = IntStream.range(0, activeUserCount).mapToObj(i -> db.users().insertUser( + u -> u.setExternalIdentityProvider("provider" + i).setLastSonarlintConnectionDate(i * 2L))) + .collect(Collectors.toList()); + + // update last connection + activeUsers.forEach(u -> db.users().updateLastConnectionDate(u, 5L)); + + UserDto inactiveUser = db.users().insertUser(u -> u.setActive(false).setExternalIdentityProvider("provider0")); userIndexer.indexAll(); MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY)); @@ -109,29 +127,37 @@ public class TelemetryDataLoaderImplTest { MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY)); MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY)); - ComponentDto project1 = db.components().insertPublicProject(); - ComponentDto project1Branch = db.components().insertProjectBranch(project1); - db.measures().insertLiveMeasure(project1, lines, m -> m.setValue(200d)); - db.measures().insertLiveMeasure(project1, ncloc, m -> m.setValue(100d)); + ComponentDto project1 = db.components().insertPrivateProject(); + db.measures().insertLiveMeasure(project1, lines, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(project1, ncloc, m -> m.setValue(110d)); db.measures().insertLiveMeasure(project1, coverage, m -> m.setValue(80d)); - db.measures().insertLiveMeasure(project1, nclocDistrib, m -> m.setValue(null).setData("java=200;js=50")); + db.measures().insertLiveMeasure(project1, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10")); - ComponentDto project2 = db.components().insertPublicProject(); - db.measures().insertLiveMeasure(project2, lines, m -> m.setValue(300d)); + ComponentDto project2 = db.components().insertPrivateProject(); + db.measures().insertLiveMeasure(project2, lines, m -> m.setValue(200d)); db.measures().insertLiveMeasure(project2, ncloc, m -> m.setValue(200d)); db.measures().insertLiveMeasure(project2, coverage, m -> m.setValue(80d)); - db.measures().insertLiveMeasure(project2, nclocDistrib, m -> m.setValue(null).setData("java=300;kotlin=2500")); - projectMeasuresIndexer.indexAll(); + db.measures().insertLiveMeasure(project2, nclocDistrib, m -> m.setValue(null).setData("java=180;js=20")); + + SnapshotDto project1Analysis = db.components().insertSnapshot(project1, t -> t.setLast(true).setBuildDate(analysisDate)); + SnapshotDto project2Analysis = db.components().insertSnapshot(project2, t -> t.setLast(true).setBuildDate(analysisDate)); + db.measures().insertMeasure(project1, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10")); + db.measures().insertMeasure(project2, project2Analysis, nclocDistrib, m -> m.setData("java=180;js=20")); + + insertAnalysisProperty(project1Analysis, "prop-uuid-1", SONAR_ANALYSIS_DETECTEDCI, "ci-1"); + insertAnalysisProperty(project2Analysis, "prop-uuid-2", SONAR_ANALYSIS_DETECTEDCI, "ci-2"); + insertAnalysisProperty(project1Analysis, "prop-uuid-3", SONAR_ANALYSIS_DETECTEDSCM, "scm-1"); + insertAnalysisProperty(project2Analysis, "prop-uuid-4", SONAR_ANALYSIS_DETECTEDSCM, "scm-2"); // alm db.almSettings().insertAzureAlmSetting(); - db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com")); - db.almSettings().insertBitbucketAlmSetting(); - db.almSettings().insertBitbucketCloudAlmSetting(); db.almSettings().insertGitHubAlmSetting(); - db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com")); - db.almSettings().insertGitlabAlmSetting(); - db.almSettings().insertGitlabAlmSetting(a -> a.setUrl("https://gitlab.com/api/v4")); + AlmSettingDto almSettingDto = db.almSettings().insertAzureAlmSetting(a -> a.setUrl("https://dev.azure.com")); + AlmSettingDto gitHubAlmSetting = db.almSettings().insertGitHubAlmSetting(a -> a.setUrl("https://api.github.com")); + db.almSettings().insertAzureProjectAlmSetting(almSettingDto, db.components().getProjectDto(project1)); + db.almSettings().insertGitlabProjectAlmSetting(gitHubAlmSetting, db.components().getProjectDto(project2)); + + projectMeasuresIndexer.indexAll(); TelemetryData data = communityUnderTest.load(); assertThat(data.getServerId()).isEqualTo(serverId); @@ -140,24 +166,39 @@ public class TelemetryDataLoaderImplTest { assertDatabaseMetadata(data.getDatabase()); assertThat(data.getPlugins()).containsOnly( entry("java", "4.12.0.11033"), entry("scmgit", "1.2"), entry("other", "undefined")); - assertThat(data.getUserCount()).isEqualTo(userCount); + assertThat(data.getUserCount()).isEqualTo(activeUserCount); assertThat(data.getProjectCount()).isEqualTo(2L); - assertThat(data.getNcloc()).isEqualTo(300L); + assertThat(data.getNcloc()).isEqualTo(310L); assertThat(data.getProjectCountByLanguage()).containsOnly( - entry("java", 2L), entry("kotlin", 1L), entry("js", 1L)); + entry("java", 2L), entry("kotlin", 1L), entry("js", 2L)); assertThat(data.getNclocByLanguage()).containsOnly( - entry("java", 500L), entry("kotlin", 2500L), entry("js", 50L)); + entry("java", 250L), entry("kotlin", 10L), entry("js", 50L)); assertThat(data.isInDocker()).isFalse(); - assertThat(data.getAlmIntegrationCountByAlm()) - .containsEntry("azure_devops_server", 1L) - .containsEntry("azure_devops_cloud", 1L) - .containsEntry("bitbucket_server", 1L) - .containsEntry("bitbucket_cloud", 1L) - .containsEntry("gitlab_server", 1L) - .containsEntry("gitlab_cloud", 1L) - .containsEntry("github_cloud", 1L) - .containsEntry("github_server", 1L); assertThat(data.getExternalAuthenticationProviders()).containsExactlyInAnyOrder("provider0", "provider1", "provider2"); + + assertThat(data.getUserTelemetries()) + .extracting(UserTelemetryDto::getUuid, UserTelemetryDto::getLastConnectionDate, UserTelemetryDto::getLastSonarlintConnectionDate, UserTelemetryDto::isActive) + .containsExactlyInAnyOrder( + tuple(activeUsers.get(0).getUuid(), lastConnectionDate, activeUsers.get(0).getLastSonarlintConnectionDate(), true), + tuple(activeUsers.get(1).getUuid(), lastConnectionDate, activeUsers.get(1).getLastSonarlintConnectionDate(), true), + tuple(activeUsers.get(2).getUuid(), lastConnectionDate, activeUsers.get(2).getLastSonarlintConnectionDate(), true), + tuple(inactiveUser.getUuid(), null, inactiveUser.getLastSonarlintConnectionDate(), false)); + assertThat(data.getProjects()) + .extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc, TelemetryData.Project::getLastAnalysis) + .containsExactlyInAnyOrder( + tuple(project1.uuid(), "java", 70L, analysisDate), + tuple(project1.uuid(), "js", 30L, analysisDate), + tuple(project1.uuid(), "kotlin", 10L, analysisDate), + tuple(project2.uuid(), "java", 180L, analysisDate), + tuple(project2.uuid(), "js", 20L, analysisDate) + ); + assertThat(data.getProjectStatistics()) + .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount, + TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi, TelemetryData.ProjectStatistics::getAlm) + .containsExactlyInAnyOrder( + tuple(1L, 0L, "scm-1", "ci-1", "azure_devops_cloud"), + tuple(1L, 0L, "scm-2", "ci-2", "github_cloud") + ); } private void assertDatabaseMetadata(TelemetryData.Database database) { @@ -205,6 +246,50 @@ public class TelemetryDataLoaderImplTest { } @Test + public void take_largest_branch_snapshot_project_data() { + server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4"); + + MetricDto lines = db.measures().insertMetric(m -> m.setKey(LINES_KEY)); + MetricDto ncloc = db.measures().insertMetric(m -> m.setKey(NCLOC_KEY)); + MetricDto coverage = db.measures().insertMetric(m -> m.setKey(COVERAGE_KEY)); + MetricDto nclocDistrib = db.measures().insertMetric(m -> m.setKey(NCLOC_LANGUAGE_DISTRIBUTION_KEY)); + + ComponentDto project = db.components().insertPublicProject(); + db.measures().insertLiveMeasure(project, lines, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(project, ncloc, m -> m.setValue(110d)); + db.measures().insertLiveMeasure(project, coverage, m -> m.setValue(80d)); + db.measures().insertLiveMeasure(project, nclocDistrib, m -> m.setValue(null).setData("java=70;js=30;kotlin=10")); + + ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setBranchType(BRANCH)); + db.measures().insertLiveMeasure(branch, lines, m -> m.setValue(180d)); + db.measures().insertLiveMeasure(branch, ncloc, m -> m.setValue(180d)); + db.measures().insertLiveMeasure(branch, coverage, m -> m.setValue(80d)); + db.measures().insertLiveMeasure(branch, nclocDistrib, m -> m.setValue(null).setData("java=100;js=50;kotlin=30")); + + SnapshotDto project1Analysis = db.components().insertSnapshot(project, t -> t.setLast(true)); + SnapshotDto project2Analysis = db.components().insertSnapshot(branch, t -> t.setLast(true)); + db.measures().insertMeasure(project, project1Analysis, nclocDistrib, m -> m.setData("java=70;js=30;kotlin=10")); + db.measures().insertMeasure(branch, project2Analysis, nclocDistrib, m -> m.setData("java=100;js=50;kotlin=30")); + + projectMeasuresIndexer.indexAll(); + + TelemetryData data = communityUnderTest.load(); + + assertThat(data.getProjects()).extracting(TelemetryData.Project::getProjectUuid, TelemetryData.Project::getLanguage, TelemetryData.Project::getLoc) + .containsExactlyInAnyOrder( + tuple(project.uuid(), "java", 100L), + tuple(project.uuid(), "js", 50L), + tuple(project.uuid(), "kotlin", 30L) + ); + assertThat(data.getProjectStatistics()) + .extracting(TelemetryData.ProjectStatistics::getBranchCount, TelemetryData.ProjectStatistics::getPullRequestCount, + TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi) + .containsExactlyInAnyOrder( + tuple(2L, 0L, "undetected", "undetected") + ); + } + + @Test public void data_contains_no_license_type_on_community_edition() { TelemetryData data = communityUnderTest.load(); @@ -351,9 +436,29 @@ public class TelemetryDataLoaderImplTest { assertThat(data.getCustomSecurityConfigs()).isEmpty(); } + @Test + public void undetected_alm_ci_slm_data() { + server.setId("AU-TpxcB-iU5OvuD2FL7").setVersion("7.5.4"); + db.components().insertPublicProject(); + projectMeasuresIndexer.indexAll(); + TelemetryData data = communityUnderTest.load(); + assertThat(data.getProjectStatistics()) + .extracting(TelemetryData.ProjectStatistics::getAlm, TelemetryData.ProjectStatistics::getScm, TelemetryData.ProjectStatistics::getCi) + .containsExactlyInAnyOrder(tuple("undetected", "undetected", "undetected")); + } + private PluginInfo newPlugin(String key, String version) { return new PluginInfo(key) .setVersion(Version.create(version)); } + private void insertAnalysisProperty(SnapshotDto snapshotDto, String uuid, String key, String value) { + db.getDbClient().analysisPropertiesDao().insert(db.getSession(), new AnalysisPropertyDto() + .setUuid(uuid) + .setAnalysisUuid(snapshotDto.getUuid()) + .setKey(key) + .setValue(value) + .setCreatedAt(1L)); + } + } |