SONAR-23308 Re-introduce cache step for GitLab tutorials & Fix the cache path/policy

3 files changed, 154 insertions, 87 deletions

diff --git a/server/sonar-web/src/main/js/components/tutorials/gitlabci/__tests__/__snapshots__/GitLabCITutorial-it.tsx.snap b/server/sonar-web/src/main/js/components/tutorials/gitlabci/__tests__/__snapshots__/GitLabCITutorial-it.tsx.snap
index 1e5cca0216d..cc13ab96cc2 100644
--- a/server/sonar-web/src/main/js/components/tutorials/gitlabci/__tests__/__snapshots__/GitLabCITutorial-it.tsx.snap
+++ b/server/sonar-web/src/main/js/components/tutorials/gitlabci/__tests__/__snapshots__/GitLabCITutorial-it.tsx.snap
@@ -8,12 +8,20 @@ variables:
   GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
 
 stages:
-  - sonarqube-check
+  - build-sonar
   - sonarqube-vulnerability-report
 
-sonarqube-check:
-  stage: sonarqube-check
+build-sonar:
+  stage: build-sonar
   
+  
+  cache:
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
+    paths:
+      - "\${SONAR_USER_HOME}/cache"
+      - sonar-scanner/
+      
   script: 
       - "dotnet tool install --global dotnet-sonarscanner"
       - "export PATH=\\"$PATH:$HOME/.dotnet/tools\\""
@@ -54,26 +62,33 @@ variables:
 
 stages:
   - get-binaries
-  - build-analyze
+  - build-sonar
   - sonarqube-vulnerability-report
 
 get-binaries:
   stage: get-binaries
+  
   cache:
-    policy: push
-    key: "\${CI_COMMIT_SHORT_SHA}"
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
     paths:
+      - "\${SONAR_USER_HOME}/cache"
       - sonar-scanner/
       - build-wrapper/
   script:
     # Download sonar-scanner
-    - curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.0.4584-linux-x64.zip'
-    - unzip -o sonar-scanner.zip
-    - mv sonar-scanner-6.2.0.4584-linux-x64 sonar-scanner
-    # Download build-wrapper
-    - curl -sSLo ./build-wrapper-linux-x86.zip "$SONAR_HOST_URL/static/cpp/build-wrapper-linux-x86.zip"
-    - unzip -o build-wrapper-linux-x86.zip
-    - mv build-wrapper-linux-x86 build-wrapper
+    - if [ ! -d sonar-scanner ]; then
+        curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.0.4584-linux-x64.zip' &&
+        unzip -o sonar-scanner.zip &&
+        mv sonar-scanner-6.2.0.4584-linux-x64 sonar-scanner;
+      fi
+    
+    # Download build wrapper
+    - if [ ! -d build-wrapper ]; then
+        curl -sSLo ./build-wrapper-linux-x86.zip "$SONAR_HOST_URL/static/cpp/build-wrapper-linux-x86.zip" &&
+        unzip -o build-wrapper-linux-x86.zip &&
+        mv build-wrapper-linux-x86 build-wrapper;
+      fi
 
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -81,17 +96,20 @@ get-binaries:
     - if: $CI_COMMIT_BRANCH == 'main'
     - if: $CI_COMMIT_BRANCH == 'develop'
 
-build:
-  stage: build-analyze
+build-sonar:
+  stage: build-sonar
   script:
     - build-wrapper/build-wrapper-linux-x86-64 --out-dir "\${BUILD_WRAPPER_OUT_DIR}" <your clean build command>
     - sonar-scanner/bin/sonar-scanner -Dsonar.host.url="\${SONAR_HOST_URL}" -Dsonar.cfamily.compile-commands="\${BUILD_WRAPPER_OUT_DIR}/compile_commands.json"
+  
   cache:
-    policy: pull
-    key: "\${CI_COMMIT_SHORT_SHA}"
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
     paths:
-    - sonar-scanner/
-    - build-wrapper/
+      - "\${SONAR_USER_HOME}/cache"
+      - sonar-scanner/
+      - build-wrapper/
+
 
 sonarqube-vulnerability-report:
   stage: sonarqube-vulnerability-report
@@ -120,22 +138,26 @@ variables:
 
 stages:
   - get-binaries
-  - sonarqube-check
+  - build-sonar
   - sonarqube-vulnerability-report
 
 get-binaries:
   stage: get-binaries
+  
   cache:
-    policy: push
-    key: "\${CI_COMMIT_SHORT_SHA}"
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
     paths:
+      - "\${SONAR_USER_HOME}/cache"
       - sonar-scanner/
       
   script:
     # Download sonar-scanner
-    - curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.0.4584-linux-x64.zip'
-    - unzip -o sonar-scanner.zip
-    - mv sonar-scanner-6.2.0.4584-linux-x64 sonar-scanner
+    - if [ ! -d sonar-scanner ]; then
+        curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.0.4584-linux-x64.zip' &&
+        unzip -o sonar-scanner.zip &&
+        mv sonar-scanner-6.2.0.4584-linux-x64 sonar-scanner;
+      fi
     
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -143,16 +165,19 @@ get-binaries:
     - if: $CI_COMMIT_BRANCH == 'main'
     - if: $CI_COMMIT_BRANCH == 'develop'
 
-sonarqube-check:
-  stage: sonarqube-check
+build-sonar:
+  stage: build-sonar
   dependencies:
     - get-binaries
+
+  
   cache:
-    policy: pull
-    key: "\${CI_COMMIT_SHORT_SHA}"
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
     paths:
+      - "\${SONAR_USER_HOME}/cache"
       - sonar-scanner/
-
+      
   script: sonar-scanner/bin/sonar-scanner -Dsonar.host.url="\${SONAR_HOST_URL}" 
   allow_failure: true
   rules:
@@ -193,22 +218,26 @@ variables:
 
 stages:
   - get-binaries
-  - sonarqube-check
+  - build-sonar
   - sonarqube-vulnerability-report
 
 get-binaries:
   stage: get-binaries
+  
   cache:
-    policy: push
-    key: "\${CI_COMMIT_SHORT_SHA}"
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
     paths:
+      - "\${SONAR_USER_HOME}/cache"
       - sonar-scanner/
       
   script:
     # Download sonar-scanner
-    - curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.0.4584-linux-x64.zip'
-    - unzip -o sonar-scanner.zip
-    - mv sonar-scanner-6.2.0.4584-linux-x64 sonar-scanner
+    - if [ ! -d sonar-scanner ]; then
+        curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.0.4584-linux-x64.zip' &&
+        unzip -o sonar-scanner.zip &&
+        mv sonar-scanner-6.2.0.4584-linux-x64 sonar-scanner;
+      fi
     
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -216,9 +245,17 @@ get-binaries:
     - if: $CI_COMMIT_BRANCH == 'main'
     - if: $CI_COMMIT_BRANCH == 'develop'
 
-sonarqube-check:
-  stage: sonarqube-check
+build-sonar:
+  stage: build-sonar
   
+  
+  cache:
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
+    paths:
+      - "\${SONAR_USER_HOME}/cache"
+      - sonar-scanner/
+      
   script: 
     - <commands to build your project>
     - sonar-scanner/bin/sonar-scanner --define sonar.host.url="\${SONAR_HOST_URL}"
@@ -260,12 +297,20 @@ variables:
   GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
 
 stages:
-  - sonarqube-check
+  - build-sonar
   - sonarqube-vulnerability-report
 
-sonarqube-check:
-  stage: sonarqube-check
+build-sonar:
+  stage: build-sonar
   
+  
+  cache:
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
+    paths:
+      - "\${SONAR_USER_HOME}/cache"
+      - sonar-scanner/
+      
   script: gradle sonar
   allow_failure: true
   rules:
@@ -327,12 +372,20 @@ variables:
   GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
 
 stages:
-  - sonarqube-check
+  - build-sonar
   - sonarqube-vulnerability-report
 
-sonarqube-check:
-  stage: sonarqube-check
+build-sonar:
+  stage: build-sonar
+  
   
+  cache:
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
+    paths:
+      - "\${SONAR_USER_HOME}/cache"
+      - sonar-scanner/
+      
   script: 
     - mvn verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
   allow_failure: true
@@ -377,12 +430,20 @@ variables:
   GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
 
 stages:
-  - sonarqube-check
+  - build-sonar
   - sonarqube-vulnerability-report
 
-sonarqube-check:
-  stage: sonarqube-check
+build-sonar:
+  stage: build-sonar
+  
   
+  cache:
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
+    paths:
+      - "\${SONAR_USER_HOME}/cache"
+      - sonar-scanner/
+      
   script: 
     - sonar-scanner -Dsonar.host.url="\${SONAR_HOST_URL}"
   allow_failure: true
diff --git a/server/sonar-web/src/main/js/components/tutorials/gitlabci/commands/PipeCommand.tsx b/server/sonar-web/src/main/js/components/tutorials/gitlabci/commands/PipeCommand.tsx
index d1b1406b0df..79c4d222828 100644
--- a/server/sonar-web/src/main/js/components/tutorials/gitlabci/commands/PipeCommand.tsx
+++ b/server/sonar-web/src/main/js/components/tutorials/gitlabci/commands/PipeCommand.tsx
@@ -28,6 +28,7 @@ import {
   getScannerUrlSuffix,
   isCFamily,
   shouldFetchBuildWrapper,
+  shouldFetchScanner,
 } from '../../utils';
 
 export interface PipeCommandProps {
@@ -95,31 +96,39 @@ const BUILD_TOOL_SPECIFIC: {
 export default function PipeCommand(props: Readonly<PipeCommandProps>) {
   const { projectKey, buildTool, config, arch } = props;
   const { autoConfig } = config;
-
   const { image, script } = BUILD_TOOL_SPECIFIC[buildTool];
 
   const suffix = getScannerUrlSuffix(OSs.Linux, arch);
   const buildWrapperFolder = getBuildWrapperFolderLinux(arch);
 
-  const getBinaries = `get-binaries:
-  stage: get-binaries
+  const cacheDefinition = `
   cache:
-    policy: push
-    key: "\${CI_COMMIT_SHORT_SHA}"
+    policy: pull-push
+    key: "sonar-cache-$CI_COMMIT_REF_SLUG"
     paths:
+      - "\${SONAR_USER_HOME}/cache"
       - sonar-scanner/
-      ${shouldFetchBuildWrapper(buildTool, autoConfig) ? `- build-wrapper/` : ''}
+      ${shouldFetchBuildWrapper(buildTool, autoConfig) ? '- build-wrapper/' : ''}`;
+
+  const getBinariesStage = `get-binaries:
+  stage: get-binaries
+  ${cacheDefinition}
   script:
     # Download sonar-scanner
-    - curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_CLI_LATEST_VERSION}${suffix}.zip'
-    - unzip -o sonar-scanner.zip
-    - mv sonar-scanner-${SONAR_SCANNER_CLI_LATEST_VERSION}${suffix} sonar-scanner
+    - if [ ! -d sonar-scanner ]; then
+        curl -sSLo ./sonar-scanner.zip 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_CLI_LATEST_VERSION}${suffix}.zip' &&
+        unzip -o sonar-scanner.zip &&
+        mv sonar-scanner-${SONAR_SCANNER_CLI_LATEST_VERSION}${suffix} sonar-scanner;
+      fi
     ${
       shouldFetchBuildWrapper(buildTool, autoConfig)
-        ? `# Download build-wrapper
-    - curl -sSLo ./${buildWrapperFolder}.zip "$SONAR_HOST_URL/static/cpp/${buildWrapperFolder}.zip"
-    - unzip -o ${buildWrapperFolder}.zip
-    - mv ${buildWrapperFolder} build-wrapper
+        ? `
+    # Download build wrapper
+    - if [ ! -d build-wrapper ]; then
+        curl -sSLo ./${buildWrapperFolder}.zip "$SONAR_HOST_URL/static/cpp/${buildWrapperFolder}.zip" &&
+        unzip -o ${buildWrapperFolder}.zip &&
+        mv ${buildWrapperFolder} build-wrapper;
+      fi
 `
         : ''
     }
@@ -129,32 +138,24 @@ export default function PipeCommand(props: Readonly<PipeCommandProps>) {
     - if: $CI_COMMIT_BRANCH == 'main'
     - if: $CI_COMMIT_BRANCH == 'develop'`;
 
-  const buildAnalyze = `build:
-  stage: build-analyze
+  const sonarWithBuildWrapperStage = `build-sonar:
+  stage: build-sonar
   script:
     - build-wrapper/${getBuildWrapperExecutableLinux(arch)} --out-dir "\${BUILD_WRAPPER_OUT_DIR}" <your clean build command>
     - ${script(projectKey, autoConfig)}
-  cache:
-    policy: pull
-    key: "\${CI_COMMIT_SHORT_SHA}"
-    paths:
-    - sonar-scanner/
-    - build-wrapper/`;
+  ${cacheDefinition}
+`;
 
-  const sonarqubeCheck = `sonarqube-check:
-  stage: sonarqube-check
+  const sonarStage = `build-sonar:
+  stage: build-sonar
   ${
     isCFamily(buildTool)
       ? `dependencies:
     - get-binaries
-  cache:
-    policy: pull
-    key: "\${CI_COMMIT_SHORT_SHA}"
-    paths:
-      - sonar-scanner/
 `
       : ''
   }
+  ${cacheDefinition}
   script: ${script(projectKey, autoConfig)}
   allow_failure: true
   rules:
@@ -163,7 +164,7 @@ export default function PipeCommand(props: Readonly<PipeCommandProps>) {
     - if: $CI_COMMIT_BRANCH == 'main'
     - if: $CI_COMMIT_BRANCH == 'develop'`;
 
-  const vulnerabilityReport = `sonarqube-vulnerability-report:
+  const vulnerabilityReportStage = `sonarqube-vulnerability-report:
   stage: sonarqube-vulnerability-report
   script:
     - 'curl -u "\${SONAR_TOKEN}:" "\${SONAR_HOST_URL}/api/issues/gitlab_sast_export?projectKey=${projectKey}&branch=\${CI_COMMIT_BRANCH}&pullRequest=\${CI_MERGE_REQUEST_IID}" -o gl-sast-sonar-report.json'
@@ -179,21 +180,19 @@ export default function PipeCommand(props: Readonly<PipeCommandProps>) {
       sast: gl-sast-sonar-report.json
 `;
 
-  let stageDeclaration = ['sonarqube-vulnerability-report'];
-  let stages = [vulnerabilityReport];
+  let stageDeclaration: string[] = [];
+  let stages: string[] = [];
 
   if (shouldFetchBuildWrapper(buildTool, autoConfig)) {
     // only for c-family languages on manual configuration
-    stages = [buildAnalyze, ...stages];
-    stageDeclaration = ['build-analyze', ...stageDeclaration];
+    stages = [getBinariesStage, sonarWithBuildWrapperStage, vulnerabilityReportStage];
+    stageDeclaration = ['get-binaries', 'build-sonar', 'sonarqube-vulnerability-report'];
+  } else if (shouldFetchScanner(buildTool)) {
+    stages = [getBinariesStage, sonarStage, vulnerabilityReportStage];
+    stageDeclaration = ['get-binaries', 'build-sonar', 'sonarqube-vulnerability-report'];
   } else {
-    stages = [sonarqubeCheck, ...stages];
-    stageDeclaration = ['sonarqube-check', ...stageDeclaration];
-  }
-
-  if (isCFamily(buildTool) || buildTool === BuildTools.Dart) {
-    stages = [getBinaries, ...stages];
-    stageDeclaration = ['get-binaries', ...stageDeclaration];
+    stages = [sonarStage, vulnerabilityReportStage];
+    stageDeclaration = ['build-sonar', 'sonarqube-vulnerability-report'];
   }
 
   const stageDefinition =
diff --git a/server/sonar-web/src/main/js/components/tutorials/utils.ts b/server/sonar-web/src/main/js/components/tutorials/utils.ts
index 8a230b701e6..12425ea5998 100644
--- a/server/sonar-web/src/main/js/components/tutorials/utils.ts
+++ b/server/sonar-web/src/main/js/components/tutorials/utils.ts
@@ -209,3 +209,10 @@ export function shouldFetchBuildWrapper(buildTool: BuildTools, autoConfig?: Auto
     buildTool === BuildTools.ObjectiveC
   );
 }
+
+export function shouldFetchScanner(buildTool: BuildTools) {
+  if (isCFamily(buildTool)) {
+    return true;
+  }
+  return [BuildTools.Dart].includes(buildTool);
+}