From: Wouter Admiraal Date: Wed, 4 Mar 2020 09:53:12 +0000 (+0100) Subject: BUILD-759 Move Jenkins + Travis QA to Cirrus CI X-Git-Tag: 4.4.0.2170~5 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=ae038e155d03502e1dcd154408f29a057f089456;p=sonar-scanner-cli.git BUILD-759 Move Jenkins + Travis QA to Cirrus CI --- diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000..3c726ef --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,146 @@ +# content of service-account-credentials.json, used to access to Google Cloud Platform +gcp_credentials: ENCRYPTED[!1fcffe5cc2de4fbbda5befae835ca275a47c1148dabbbb7b5d21334604efba3ef8730d8bc5820952e575c7dd5177e433!] + +# +# ENV VARIABLES +# +env: + ### Shared variables + ARTIFACTORY_URL: ENCRYPTED[!2f8fa307d3289faa0aa6791f18b961627ae44f1ef46b136e1a1e63b0b4c86454dbb25520d49b339e2d50a1e1e5f95c88!] + ARTIFACTORY_PRIVATE_USERNAME: private-reader + ARTIFACTORY_PRIVATE_PASSWORD: ENCRYPTED[!921e2792ce1fc164aaea1146ab2478e7aefd8aaa87022ca745adccee4deaa470bb883ad3066738fceb37622f239296a7!] + ARTIFACTORY_API_KEY: ENCRYPTED[!f9526e763214dcd6f0cac5fbf712664bcf05395cc5c8f9f122ccf3a4c6d42c2bc809bff8a6d9904c935a47a05676682c!] + ARTIFACTORY_DEPLOY_USERNAME: public-qa-deployer + ARTIFACTORY_DEPLOY_PASSWORD: ENCRYPTED[!ee8f6410a42b81c6c91c7b760e7be7796a7774b6e6e6acf3ff2ecf8c7c04c732865a0300673ea41155c0d19a989c0a5a!] + ARTIFACTORY_DEPLOY_REPO: sonarsource-public-qa + + GCF_ACCESS_TOKEN: ENCRYPTED[!1fb91961a5c01e06e38834e55755231d649dc62eca354593105af9f9d643d701ae4539ab6a8021278b8d9348ae2ce8be!] + PROMOTE_URL: ENCRYPTED[!e22ed2e34a8f7a1aea5cff653585429bbd3d5151e7201022140218f9c5d620069ec2388f14f83971e3fd726215bc0f5e!] + + GITHUB_TOKEN: ENCRYPTED[!f272985ea5b49b3cf9c414b98de6a8e9096be47bfcee52f33311ba3131a2af637c1b956f49585b7757dd84b7c030233a!] + + BURGR_URL: ENCRYPTED[!c7e294da94762d7bac144abef6310c5db300c95979daed4454ca977776bfd5edeb557e1237e3aa8ed722336243af2d78!] + BURGR_USERNAME: ENCRYPTED[!b29ddc7610116de511e74bec9a93ad9b8a20ac217a0852e94a96d0066e6e822b95e7bc1fe152afb707f16b70605fddd3!] + BURGR_PASSWORD: ENCRYPTED[!83e130718e92b8c9de7c5226355f730e55fb46e45869149a9223e724bb99656878ef9684c5f8cfef434aa716e87f4cf2!] + + ### Project variables + DEPLOY_PULL_REQUEST: true + ARTIFACTS: org.sonarsource.scanner.cli:sonar-scanner-cli:jar + + +# +# RE-USABLE CONFIGS +# +container_definition: &CONTAINER_DEFINITION + image: gcr.io/ci-cd-215716/base:latest + cluster_name: cirrus-uscentral1a-cluster + zone: us-central1-a + namespace: default + +only_sonarsource_qa: &ONLY_SONARSOURCE_QA + only_if: $CIRRUS_USER_COLLABORATOR == 'true' && ($CIRRUS_PR != "" || $CIRRUS_BRANCH == "master" || $CIRRUS_BRANCH =~ "branch-.*" || $CIRRUS_BRANCH =~ "dogfood-on-.*") + + +# +# TASKS +# +build_task: + gke_container: + <<: *CONTAINER_DEFINITION + cpu: 1 + memory: 2G + env: + SONAR_TOKEN: ENCRYPTED[!5ba7cbb5bf9d168de69bcd444d9e884c9cf664be1115640cc64e49df6d241c309a87fc527cab533c08f289b167187017!] + SONAR_HOST_URL: https://next.sonarqube.com/sonarqube + maven_cache: + folder: ${CIRRUS_WORKING_DIR}/.m2/repository + script: + - source cirrus-env BUILD + - regular_mvn_build_deploy_analyze -Pdist-linux,dist-windows,dist-macosx + cleanup_before_cache_script: + - cleanup_maven_repository + +linux_qa_task: + depends_on: + - build + <<: *ONLY_SONARSOURCE_QA + gke_container: + dockerfile: it/docker/Dockerfile + builder_image_project: ci-cd-215716 + builder_image_name: docker-builder-v2 + cluster_name: cirrus-uscentral1a-cluster + zone: us-central1-a + namespace: default + cpu: 1 + memory: 2G + env: + matrix: + - SQ_VERSION: LATEST_RELEASE[7.9] + - SQ_VERSION: DEV + maven_cache: + folder: ${CIRRUS_WORKING_DIR}/.m2/repository + qa_script: + - source cirrus-env QA + - source set_maven_build_version $BUILD_NUMBER + - cd it + - mvn -B -e -Dsonar.runtimeVersion="$SQ_VERSION" -Dmaven.test.redirectTestOutputToFile=false verify + cleanup_before_cache_script: + - cleanup_maven_repository + +create_win_vm_task: + <<: *ONLY_SONARSOURCE_QA + skip: "!changesInclude('it/packer/setup.ps1', 'it/packer/sonar-scanner-cli-qa.json')" + gce_instance: + image_project: ci-cd-215716 + image_name: packer-builder-v1 + zone: us-central1-a + type: n1-standard-8 + use_ssd: true + scopes: + - cloud-platform + build_script: + - packer build -force it/packer/sonar-scanner-cli-qa.json + +win_qa_task: + depends_on: + - create_win_vm + - build + <<: *ONLY_SONARSOURCE_QA + gce_instance: + image_project: ci-cd-215716 + # This VM is built using Packer, see "create_win_vm_task" above. + image_name: sonar-scanner-cli-qa + platform: windows + zone: us-central1-a + type: n1-standard-8 + use_ssd: true + env: + CIRRUS_SHELL: bash + matrix: + - SQ_VERSION: LATEST_RELEASE[7.9] + - SQ_VERSION: DEV + maven_cache: + folder: ${CIRRUS_WORKING_DIR}/.m2/repository + qa_script: + - source cirrus-env QA + - source set_maven_build_version $BUILD_NUMBER + - cd it + - mvn -B -e -Dsonar.runtimeVersion="$SQ_VERSION" -Dmaven.test.redirectTestOutputToFile=false verify + cleanup_before_cache_script: + - cleanup_maven_repository + +promote_task: + depends_on: + - linux_qa + - win_qa + <<: *ONLY_SONARSOURCE_QA + gke_container: + <<: *CONTAINER_DEFINITION + cpu: 0.5 + memory: 500M + maven_cache: + folder: $CIRRUS_WORKING_DIR/.m2/repository + script: + - cirrus_promote_maven + cleanup_before_cache_script: + - cleanup_maven_repository diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2631e78..0000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: java -sudo: false - -jdk: openjdk11 - -install: true -script: ./travis.sh - -cache: - directories: - - $HOME/.m2/repository - - $HOME/.sonar - -notifications: - email: false - webhooks: - - secure: "KmAKurBY49QiBEePmi2G54NN+i9h5uweiQH32CF0DBKsBB7Coswwp4ULdImYGz0QS/r/xIhA1dOzyq7uQ+tmjQsL0DGoUYXlm6P1KO/oQ8UbLQOxIrNN+tH+QxdzDZwIcpvxFUFiCN8bG8XiXSAyKmHeWVNwYKcsZI6GkacNu307h6FOveKqstGfQJQWUObooMGoaiHhdq7LGd25caWCN2QX+mqnjpwGK1wuY5OLABrU3GJnchJj7DjovjiGmstRCDFf+1Xmq+rJ6Gq2hp9YhNQwYgOIlWGqaNzHt6pdz/Wa/QO92N196FPWg/DAidO5esXJUsu4qCwcMHsatGnyOoMHPsllB7Ob12WCeQJgRk6RifeWzt13mmw0mnnU22ZrtwldZpw3cL5eqT5z0ZxLHuHeNP5wiFVFImKuu/GH6tF1gN8fAerDPfiRDNkYWo6VH31ZqqUOBDWKne2hbe17SGwv0PhKq8nuBpKZreRqRZ/xVSN/UcTgVr065UHHxqon+mbglw8C88aTbUuT05O1V9sMl6D6xObEmbSgLqpIpPtrOCkfDhbld7QeAJGMZuVxs502VNxUxWgd0vlp2a5S3E6WV8B6CvQJeeWUugzPR3q9oVQJYUCj01BmSO7i0wj/L6beGIc4KwEbg19Lxhy5YQHumVRve7vXGEIudLkUIFk=" - on_start: always diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index a01e687..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,136 +0,0 @@ -@Library('SonarSource@2.2') _ -pipeline { - agent { - label 'linux' - } - parameters { - string(name: 'GIT_SHA1', description: 'Git SHA1 (provided by travisci hook job)') - string(name: 'CI_BUILD_NAME', defaultValue: 'sonar-scanner-cli', description: 'Build Name (provided by travisci hook job)') - string(name: 'CI_BUILD_NUMBER', description: 'Build Number (provided by travisci hook job)') - string(name: 'GITHUB_BRANCH', defaultValue: 'master', description: 'Git branch (provided by travisci hook job)') - string(name: 'GITHUB_REPOSITORY_OWNER', defaultValue: 'SonarSource', description: 'Github repository owner(provided by travisci hook job)') - } - environment { - SONARSOURCE_QA = 'true' - MAVEN_TOOL = 'Maven 3.6.x' - JDK_VERSION = 'Java 11' - } - stages { - stage('Notify') { - steps { - sendAllNotificationQaStarted() - } - } - stage('QA') { - parallel { - stage('DOGFOOD/linux') { - agent { - label 'linux' - } - steps { - runITs("DOGFOOD") - } - } - stage('LATEST_RELEASE[6.7]/linux') { - agent { - label 'linux' - } - environment { - JDK_VERSION = 'Java 8' - } - steps { - runITs("LATEST_RELEASE[6.7]") - } - } - stage('LATEST_RELEASE/linux') { - agent { - label 'linux' - } - steps { - runITs("LATEST_RELEASE") - } - } - - stage('DOGFOOD/windows') { - agent { - label 'windows' - } - steps { - runITs("DOGFOOD") - } - } - stage('LATEST_RELEASE[6.7]/windows') { - agent { - label 'windows' - } - environment { - JDK_VERSION = 'Java 8' - } - steps { - runITs("LATEST_RELEASE[6.7]") - } - } - stage('LATEST_RELEASE/windows') { - agent { - label 'windows' - } - steps { - runITs("LATEST_RELEASE") - } - } - - stage('DOGFOOD/macosx') { - agent { - label 'macosx' - } - steps { - runITs("DOGFOOD") - } - } - stage('LATEST_RELEASE[6.7]/macosx') { - agent { - label 'macosx' - } - environment { - JDK_VERSION = 'Java 8' - } - steps { - runITs("LATEST_RELEASE[6.7]") - } - } - stage('LATEST_RELEASE/macosx') { - agent { - label 'macosx' - } - steps { - runITs("LATEST_RELEASE") - } - } - } - post { - always { - sendAllNotificationQaResult() - } - } - - } - stage('Promote') { - steps { - repoxPromoteBuild() - } - post { - always { - sendAllNotificationPromote() - } - } - } - } -} - -def runITs(SQ_VERSION) { - withMaven(maven: MAVEN_TOOL) { - dir("it") { - runMavenOrch(JDK_VERSION,"verify -Dsonar.runtimeVersion=$SQ_VERSION -U") - } - } -} \ No newline at end of file diff --git a/it/docker/Dockerfile b/it/docker/Dockerfile new file mode 100644 index 0000000..78df5f3 --- /dev/null +++ b/it/docker/Dockerfile @@ -0,0 +1,21 @@ +#------------------------------------------------------------------------------ +# Installs NodeJS, which is needed for running the Linux ITs. +# +# Build from the basedir: +# docker build -f it/docker/Dockerfile-qa -t sonar-scanner-cli-qa it/docker +# +# Verify the content of the image by running a shell session in it: +# docker run -it sonar-scanner-cli-qa bash +# +# CirrusCI builds the image when needed. No need to manually upload it to +# Google Cloud Container Registry. See section "gke_container" of .cirrus.yml +#------------------------------------------------------------------------------ + +FROM gcr.io/ci-cd-215716/base:latest + +USER root + +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - +RUN apt-get install -y nodejs + +USER sonarsource diff --git a/it/packer/README.md b/it/packer/README.md new file mode 100644 index 0000000..57dcf88 --- /dev/null +++ b/it/packer/README.md @@ -0,0 +1,23 @@ +Custom Windows VM image for sonar-scanner-cli Windows ITs +========================================================= + +This defines a custom Windows image necessary for the ITs. It contains all [build tools helpers](https://github.com/SonarSource/buildTools/blob/docker/bin/), as well as Node JS, which is needed to scan the example projects. + +How to build this VM image +-------------------------- + +*This isn't supposed to be built by hand.* We have a special image on our Google Cloud project, called *packer-builder-v1*. This image gets started up by Cirrus CI in the `create_win_vm_task` (see [`../../.cirrus.yml`](../../.cirrus.yml)), and will use [Packer](https://packer.io/) to create our custom VM image. The Packer instructions are contained in the `sonar-scanner-cli-qa.json` file. + +Note that this image is rebuilt by Cirrus CI every time the `sonar-scanner-cli-qa.json` or `setup.ps1` files change (see the `create_win_vm_task`'s `skip` instruction in [`../../.cirrus.yml`](../../.cirrus.yml)). If no changes are detected, the build will be skipped, and the previously existing image will be used. + +How to debug this VM image +-------------------------- + +1. Log on to [Google Cloud](http://console.cloud.google.com/) +2. Go to our SonarQube project (ci-cd-215716) +3. Under *Compute Engine > Images*, you should see *packer-builder-v1*. Start a new VM with this image. + This image is pre-configured for using Packer, as well as pushing new VM images to our SonarQube project. +4. Once started, SSH into this VM (you can do this directly via the browser). +5. `sudo su` to use the root user (which is configured to use the GCE service account). + You can now add packer JSON files, and run the `packer build` command to test your new images. **Make sure you remove any test images from GCE.** + diff --git a/it/packer/setup.ps1 b/it/packer/setup.ps1 new file mode 100644 index 0000000..bfaca02 --- /dev/null +++ b/it/packer/setup.ps1 @@ -0,0 +1,60 @@ +$ErrorActionPreference = 'Stop' + +function Install-Chocolatey { + # Run the installer. + Set-ExecutionPolicy Bypass -Scope Process -Force; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +} + +function Install-NodeJs { + choco install -y nodejs +} + +function Install-Buildtools { + $path = "${env:Temp}\buildTools.zip" + + # Fetch the build tools archive. + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + (New-Object System.Net.WebClient).DownloadFile('https://github.com/SonarSource/buildTools/archive/docker.zip', $path) + + # Extract the archive to the C drive. + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($path, 'C:\') + + # Update global PATH. + $currentPath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).Path + $updatedPath = $currentPath+';C:\buildTools-docker\bin' + Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $updatedPath + + # Remove archive. + del $path +} + +function Install-Maven { + choco install -y openjdk11 --version 11.0.4.11 + choco install -y maven --version 3.6.2 +} + +function Install-Git { + # We use Git to enable Unix Tools. This will allow us to use Bash-style + # commands in .cirrus.yml, like "source". + choco install -y git --version 2.23.0 --package-parameters "/GitAndUnixToolsOnPath" +} + +Write-Host "Install chocolatey" +Install-Chocolatey + +Write-Host "Install Maven" +Install-Maven + +Write-Host "Install NodeJs" +Install-NodeJs + +Write-Host "Install Unix Tools" +Install-Git + +Write-Host "Set up build tools" +Install-Buildtools + +# Disable antivirus analysis on C drive. +Write-Host "Finalize VM configuration" +Set-MpPreference -ScanAvgCPULoadFactor 5 -ExclusionPath "C:\" diff --git a/it/packer/sonar-scanner-cli-qa.json b/it/packer/sonar-scanner-cli-qa.json new file mode 100644 index 0000000..8ab3e86 --- /dev/null +++ b/it/packer/sonar-scanner-cli-qa.json @@ -0,0 +1,29 @@ +{ + "builders": [ + { + "type": "googlecompute", + "project_id": "ci-cd-215716", + "source_image": "windows-server-1809-dc-core-v20200211", + "image_name": "sonar-scanner-cli-qa", + "disk_size": "50", + "machine_type": "n1-standard-1", + "communicator": "winrm", + "winrm_username": "packer_user", + "winrm_insecure": true, + "winrm_use_ssl": true, + "metadata": { + "windows-startup-script-cmd": "winrm quickconfig -quiet & net user /add packer_user & net localgroup administrators packer_user /add & winrm set winrm/config/service/auth @{Basic=\"true\"}" + }, + "zone": "us-central1-a", + "tags": ["packer"] + } + ], + "provisioners": [ + { + "type": "powershell", + "scripts": [ + "{{template_dir}}/setup.ps1" + ] + } + ] +} diff --git a/it/src/test/java/com/sonarsource/scanner/it/ScannerTestCase.java b/it/src/test/java/com/sonarsource/scanner/it/ScannerTestCase.java index 086cba9..f03e757 100644 --- a/it/src/test/java/com/sonarsource/scanner/it/ScannerTestCase.java +++ b/it/src/test/java/com/sonarsource/scanner/it/ScannerTestCase.java @@ -68,13 +68,16 @@ public abstract class ScannerTestCase { if (StringUtils.isNotBlank(scannerVersion)) { LOG.info("Use provided Scanner version: " + scannerVersion); artifactVersion = Version.create(scannerVersion); + } else if (StringUtils.isNotBlank(System.getenv("PROJECT_VERSION"))) { + scannerVersion = System.getenv("PROJECT_VERSION"); + LOG.info("Use Scanner version from environment: " + scannerVersion); + artifactVersion = Version.create(scannerVersion); } else { try (FileInputStream fis = new FileInputStream( new File("../target/maven-archiver/pom.properties"))) { Properties props = new Properties(); props.load(fis); artifactVersion = Version.create(props.getProperty("version")); - return artifactVersion; } catch (IOException e) { throw new IllegalStateException(e); } diff --git a/run_integration_tests.sh b/run_integration_tests.sh deleted file mode 100644 index 69ddbf2..0000000 --- a/run_integration_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# Run integration tests with a given version of SonarQube -# Usage: run_integration_tests.sh "5.2" - -set -euo pipefail - -# required version of SonarQube -SONARQUBE_VERSION=$1 -shift - -cd it -mvn verify -Dsonar.runtimeVersion=$SONARQUBE_VERSION -e -B -V -U $* diff --git a/travis.sh b/travis.sh deleted file mode 100755 index e53cfba..0000000 --- a/travis.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -v - -set -euo pipefail - -function configureTravis { - mkdir -p ~/.local - curl -sSL https://github.com/SonarSource/travis-utils/tarball/v57 | tar zx --strip-components 1 -C ~/.local - source ~/.local/bin/install -} -configureTravis - -export DEPLOY_PULL_REQUEST=true - -# When a pull request is open on the branch, then the job related -# to the branch does not need to be executed and should be canceled. -# It does not book slaves for nothing. -# @TravisCI please provide the feature natively, like at AppVeyor or CircleCI ;-) -cancel_branch_build_with_pr || exit 0 - -regular_mvn_build_deploy_analyze -Pdist-linux,dist-windows,dist-macosx -