diff options
| -rwxr-xr-x | scripts/generate_batch_reports.sh | 53 | ||||
| -rwxr-xr-x | scripts/replay_batch.sh | 15 | ||||
| -rw-r--r-- | sonar-batch/src/main/java/org/sonar/batch/report/ReportPublisher.java | 56 |
3 files changed, 116 insertions, 8 deletions
diff --git a/scripts/generate_batch_reports.sh b/scripts/generate_batch_reports.sh new file mode 100755 index 00000000000..227d734b923 --- /dev/null +++ b/scripts/generate_batch_reports.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +############################## +# +# Generates batch reports dumps of a specific project into a given dump repository +# from the Git history of that project. +# +# This script runs the analysis of the project in the current directory of the N +# last commits of the current project. N can be configured. +# +# Batch report dumps can be processed with the replay_batch.sh script +# +# +############################## + + +set -euo pipefail + +DUMP_DIR="/tmp/batch_dumps" +HISTORY_LENGTH=30 +SONAR_HOST="http://localhost:9000" +SONAR_USER="admin" +SONAR_PASSWORD="admin" +SONAR_JDBC_URL="jdbc:postgresql://localhost:5432/sonar" +SONAR_JDBC_USERNAME="sonar" +SONAR_JDBC_PASSWORD="sonar" + +function clean() { + git co $branch_name + git branch -D working_branch +} + +trap "clean" EXIT + +# retrieve ${HISTORY_LENGTH} commits for the current directory +git log -${HISTORY_LENGTH} --pretty=%h -- . | tac > /tmp/sha1s.txt + +branch_name=$(git symbolic-ref -q HEAD) +branch_name=${branch_name##refs/heads/} +branch_name=${branch_name:-HEAD} + +git co -b working_branch +while read sha1; do + echo $sha1 + git reset --hard $sha1 + date=`git show -s --format=%ci | sed 's/ /T/' | sed 's/ //'` + + echo "" + echo "======================== analyzing at $date ($sha1) =======================================" + $M2_HOME/bin/mvn sonar:sonar -Dsonar.host.url=$SONAR_HOST -Dsonar.login=$SONAR_USER -Dsonar.password=$SONAR_PASSWORD -Dsonar.analysis.mode=analysis -Dsonar.issuesReport.html.enable= -Dsonar.issuesReport.console.enable= -Dsonar.jdbc.url=$SONAR_JDBC_URL -Dsonar.jdbc.username=$SONAR_JDBC_USERNAME -Dsonar.jdbc.password=$SONAR_JDBC_PASSWORD -Dsonar.batch.dumpReportDir=$DUMP_DIR -Dsonar.projectDate=$date + +done < /tmp/sha1s.txt + diff --git a/scripts/replay_batch.sh b/scripts/replay_batch.sh new file mode 100755 index 00000000000..c1bd9825016 --- /dev/null +++ b/scripts/replay_batch.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +DUMP_DIR="/tmp/batch_dumps" +SQ_ROOT_URL="http://localhost:9000" + +cd $DUMP_DIR +for file in *.zip; do + base=${file%.zip} + url=$(cat ${base}.txt) + echo "base=$base, url=$url" + + curl -u admin:admin -F report=@$DUMP_DIR/${base}.zip ${SQ_ROOT_URL}${url} +done diff --git a/sonar-batch/src/main/java/org/sonar/batch/report/ReportPublisher.java b/sonar-batch/src/main/java/org/sonar/batch/report/ReportPublisher.java index 5660e8fc23f..0f60fe4f876 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/report/ReportPublisher.java +++ b/sonar-batch/src/main/java/org/sonar/batch/report/ReportPublisher.java @@ -22,9 +22,12 @@ package org.sonar.batch.report; import com.github.kevinsawicki.http.HttpRequest; import com.google.common.annotations.VisibleForTesting; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; +import java.util.Date; import org.apache.commons.io.FileUtils; import org.picocontainer.Startable; import org.slf4j.Logger; @@ -41,11 +44,14 @@ import org.sonar.batch.bootstrap.ServerClient; import org.sonar.batch.protocol.output.BatchReportWriter; import org.sonar.batch.scan.ImmutableProjectReactor; +import static java.lang.String.format; + @BatchSide public class ReportPublisher implements Startable { private static final Logger LOG = LoggerFactory.getLogger(ReportPublisher.class); public static final String KEEP_REPORT_PROP_KEY = "sonar.batch.keepReport"; + public static final String DUMP_REPORT_PROP_KEY = "sonar.batch.dumpReportDir"; private final ServerClient serverClient; private final Server server; @@ -98,7 +104,7 @@ public class ReportPublisher implements Startable { if (!analysisMode.isPreview()) { File report = prepareReport(); if (!analysisMode.isMediumTest()) { - uploadMultiPartReport(report); + sendOrDumpReport(report); } } logSuccess(LoggerFactory.getLogger(getClass())); @@ -125,38 +131,72 @@ public class ReportPublisher implements Startable { } @VisibleForTesting - void uploadMultiPartReport(File report) { + void sendOrDumpReport(File report) { + ProjectDefinition projectDefinition = projectReactor.getRoot(); + String effectiveKey = projectDefinition.getKeyWithBranch(); + String relativeUrl = "/api/computation/submit_report?projectKey=" + effectiveKey + "&projectName=" + ServerClient.encodeForUrl(projectDefinition.getName()); + + String dumpDirLocation = settings.getString(DUMP_REPORT_PROP_KEY); + if (dumpDirLocation == null) { + uploadMultiPartReport(report, relativeUrl); + } else { + dumpReport(dumpDirLocation, effectiveKey, relativeUrl, report); + } + } + + private void uploadMultiPartReport(File report, String relativeUrl) { LOG.debug("Publish results"); long startTime = System.currentTimeMillis(); URL url; try { - ProjectDefinition projectDefinition = projectReactor.getRoot(); - String effectiveKey = projectDefinition.getKeyWithBranch(); - url = new URL(serverClient.getURL() + "/api/computation/submit_report?projectKey=" + effectiveKey + "&projectName=" + ServerClient.encodeForUrl(projectDefinition.getName())); + url = new URL(serverClient.getURL() + relativeUrl); } catch (MalformedURLException e) { throw new IllegalArgumentException("Invalid URL", e); } HttpRequest request = HttpRequest.post(url); request.trustAllCerts(); request.trustAllHosts(); - request.header("User-Agent", String.format("SonarQube %s", server.getVersion())); + request.header("User-Agent", format("SonarQube %s", server.getVersion())); request.basic(serverClient.getLogin(), serverClient.getPassword()); request.part("report", null, "application/octet-stream", report); if (!request.ok()) { int responseCode = request.code(); if (responseCode == 401) { - throw new IllegalStateException(String.format(serverClient.getMessageWhenNotAuthorized(), CoreProperties.LOGIN, CoreProperties.PASSWORD)); + throw new IllegalStateException(format(serverClient.getMessageWhenNotAuthorized(), CoreProperties.LOGIN, CoreProperties.PASSWORD)); } if (responseCode == 403) { // SONAR-4397 Details are in response content throw new IllegalStateException(request.body()); } - throw new IllegalStateException(String.format("Fail to execute request [code=%s, url=%s]: %s", responseCode, url, request.body())); + throw new IllegalStateException(format("Fail to execute request [code=%s, url=%s]: %s", responseCode, url, request.body())); } long stopTime = System.currentTimeMillis(); LOG.info("Analysis reports sent to server in " + (stopTime - startTime) + "ms"); } + private void dumpReport(String dumpDirLocation, String projectKey, String relativeUrl, File report) { + LOG.debug("Dump report to file"); + try { + dumpReportImpl(dumpDirLocation, projectKey, relativeUrl, report); + } catch (IOException | URISyntaxException e) { + LOG.error("Failed to dump report to directory " + dumpDirLocation, e); + } + } + + private void dumpReportImpl(String dumpDirLocation, String projectKey, String relativeUrl, File report) throws IOException, URISyntaxException { + File dumpDir = new File(dumpDirLocation); + if (!dumpDir.exists() || !dumpDir.isDirectory()) { + LOG.warn("Report dump directory '{}' does not exist or is not a directory", dumpDirLocation); + return; + } + long dateTime = new Date().getTime(); + File dumpedZip = new File(dumpDir, format("batch-report_%s_%s.zip", projectKey, dateTime)); + FileUtils.copyFile(report, new FileOutputStream(dumpedZip)); + File dumpedMetadata = new File(dumpDir, format("batch-report_%s_%s.txt", projectKey, dateTime)); + FileUtils.write(dumpedMetadata, relativeUrl); + LOG.info("Batch report dumped to {}", dumpedZip.getAbsolutePath()); + } + @VisibleForTesting void logSuccess(Logger logger) { if (analysisMode.isPreview() || analysisMode.isMediumTest()) { |