SONAR-12186 Fix SSF-78

server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java

@@ -122,7 +122,7 @@ public class ComputeEngineContainerImplTest {
       );
       assertThat(picoContainer.getParent().getParent().getComponentAdapters()).hasSize(
         CONTAINER_ITSELF
-          + 22 // MigrationConfigurationModule
+          + 23 // MigrationConfigurationModule
           + 17 // level 2
       );
       assertThat(picoContainer.getParent().getParent().getParent().getComponentAdapters()).hasSize(

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/MigrationConfigurationModule.java

@@ -42,6 +42,7 @@ import org.sonar.server.platform.db.migration.version.v75.DbVersion75;
 import org.sonar.server.platform.db.migration.version.v76.DbVersion76;
 import org.sonar.server.platform.db.migration.version.v77.DbVersion77;
 import org.sonar.server.platform.db.migration.version.v78.DbVersion78;
+import org.sonar.server.platform.db.migration.version.v79.DbVersion79;
 
 public class MigrationConfigurationModule extends Module {
   @Override
@@ -67,6 +68,7 @@ public class MigrationConfigurationModule extends Module {
       DbVersion76.class,
       DbVersion77.class,
       DbVersion78.class,
+      DbVersion79.class,
 
       // migration steps
       MigrationStepRegistryImpl.class,

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79.java

@@ -0,0 +1,32 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v79;
+
+import org.sonar.server.platform.db.migration.step.MigrationStepRegistry;
+import org.sonar.server.platform.db.migration.version.DbVersion;
+
+public class DbVersion79 implements DbVersion {
+  @Override
+  public void addSteps(MigrationStepRegistry registry) {
+    registry
+      .add(2800, "Truncate environment variables and system properties from existing scanner reports",
+        TruncateEnvAndSystemVarsFromScannerContext.class);
+  }
+}

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v79/TruncateEnvAndSystemVarsFromScannerContext.java

@@ -0,0 +1,59 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v79;
+
+  import java.nio.charset.StandardCharsets;
+  import java.sql.SQLException;
+  import org.sonar.db.Database;
+  import org.sonar.server.platform.db.migration.SupportsBlueGreen;
+  import org.sonar.server.platform.db.migration.step.DataChange;
+  import org.sonar.server.platform.db.migration.step.MassUpdate;
+  import org.sonar.server.platform.db.migration.step.Select;
+  import org.sonar.server.platform.db.migration.step.SqlStatement;
+
+@SupportsBlueGreen
+public class TruncateEnvAndSystemVarsFromScannerContext extends DataChange {
+
+  public TruncateEnvAndSystemVarsFromScannerContext(Database db) {
+    super(db);
+  }
+
+  @Override
+  protected void execute(Context context) throws SQLException {
+    MassUpdate massUpdate = context.prepareMassUpdate();
+    massUpdate.select("select task_uuid, context_data from ce_scanner_context sc");
+    massUpdate.update("update ce_scanner_context set context_data = ? where task_uuid = ?");
+    massUpdate.rowPluralName("truncate scanner context content");
+    massUpdate.execute(TruncateEnvAndSystemVarsFromScannerContext::truncateScannerContext);
+  }
+
+  private static boolean truncateScannerContext(Select.Row row, SqlStatement update) throws SQLException {
+    String taskUuid = row.getString(1);
+    byte[] bytes = row.getBytes(2);
+    String reportContent = new String(bytes, StandardCharsets.UTF_8);
+    int startIndex = reportContent.indexOf("SonarQube plugins:");
+    if (startIndex != -1) {
+      reportContent = reportContent.substring(startIndex);
+    }
+    update.setBytes(1, reportContent.getBytes(StandardCharsets.UTF_8));
+    update.setString(2, taskUuid);
+    return true;
+  }
+}

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/MigrationConfigurationModuleTest.java

@@ -37,7 +37,7 @@ public class MigrationConfigurationModuleTest {
     assertThat(container.getPicoContainer().getComponentAdapters())
       .hasSize(COMPONENTS_IN_EMPTY_COMPONENT_CONTAINER
         // DbVersion classes
-        + 19
+        + 20
         // Others
         + 3);
   }

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/DbVersion79Test.java

@@ -0,0 +1,41 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v79;
+
+import org.junit.Test;
+import org.sonar.server.platform.db.migration.version.DbVersion;
+
+import static org.sonar.server.platform.db.migration.version.DbVersionTestUtils.verifyMigrationCount;
+import static org.sonar.server.platform.db.migration.version.DbVersionTestUtils.verifyMinimumMigrationNumber;
+
+public class DbVersion79Test {
+  private DbVersion underTest = new DbVersion79();
+
+  @Test
+  public void migrationNumber_starts_at_2800() {
+    verifyMinimumMigrationNumber(underTest, 2800);
+  }
+
+  @Test
+  public void verify_migration_count() {
+    verifyMigrationCount(underTest, 1);
+  }
+
+}

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v79/TruncateEnvAndSystemVarsFromScannerContextTest.java

@@ -0,0 +1,113 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v79;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Blob;
+import java.sql.SQLException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonar.db.CoreDbTester;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TruncateEnvAndSystemVarsFromScannerContextTest {
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+  @Rule
+  public CoreDbTester db = CoreDbTester.createForSchema(TruncateEnvAndSystemVarsFromScannerContextTest.class, "ce_scanner_context.sql");
+
+  private TruncateEnvAndSystemVarsFromScannerContext underTest = new TruncateEnvAndSystemVarsFromScannerContext(db.database());
+
+  private static final String ENTRY_WITH_ENV_AND_SYSTEM_VARS = "Environment variables:\n" +
+    "  - PATH=blablahblah\n" +
+    "System properties:\n" +
+    "  - java.class.version=55.0\n" +
+    "\n" +
+    "  - user.language=en\n" +
+    "SonarQube plugins:\n" +
+    "  - SonarCSS 1.0.3.724 (cssfamily)\n" +
+    "Global server settings:\n" +
+    "  - sonar.core.startTime=2019-06-18T12:53:09+0200\n" +
+    "Project server settings:\n" +
+    "Project scanner properties:\n" +
+    "  - sonar.verbose=true";
+
+  private static final String TRUNCATED_ENTRY = "SonarQube plugins:\n" +
+    "  - SonarCSS 1.0.3.724 (cssfamily)\n" +
+    "Global server settings:\n" +
+    "  - sonar.core.startTime=2019-06-18T12:53:09+0200\n" +
+    "Project server settings:\n" +
+    "Project scanner properties:\n" +
+    "  - sonar.verbose=true";
+
+  @Test
+  public void execute() throws SQLException {
+    insert_entry("887f07ff-91bb-4e62-8e4b-660d5b40b16a", ENTRY_WITH_ENV_AND_SYSTEM_VARS);
+    insert_entry("9d2429cc-155a-4327-8ed7-48fdbe93edd0", TRUNCATED_ENTRY);
+
+    underTest.execute();
+
+    Map<String, Object> result = db.select("select TASK_UUID, CONTEXT_DATA from CE_SCANNER_CONTEXT")
+      .stream()
+      .collect(HashMap::new, (m, v) -> m.put((String) v.get("TASK_UUID"), readBlob((Blob) v.get("CONTEXT_DATA"))), HashMap::putAll);
+    assertThat(result).hasSize(2);
+    assertThat(result.get("887f07ff-91bb-4e62-8e4b-660d5b40b16a")).isEqualTo(TRUNCATED_ENTRY);
+    assertThat(result.get("9d2429cc-155a-4327-8ed7-48fdbe93edd0")).isEqualTo(TRUNCATED_ENTRY);
+  }
+
+  @Test
+  public void migration_is_reentrant() throws SQLException {
+    insert_entry("887f07ff-91bb-4e62-8e4b-660d5b40b16a", ENTRY_WITH_ENV_AND_SYSTEM_VARS);
+    insert_entry("9d2429cc-155a-4327-8ed7-48fdbe93edd0", TRUNCATED_ENTRY);
+
+    underTest.execute();
+    underTest.execute();
+
+    Map<String, Object> result = db.select("select TASK_UUID, CONTEXT_DATA from CE_SCANNER_CONTEXT")
+      .stream()
+      .collect(HashMap::new, (m, v) -> m.put((String) v.get("TASK_UUID"), readBlob((Blob) v.get("CONTEXT_DATA"))), HashMap::putAll);
+    assertThat(result).hasSize(2);
+    assertThat(result.get("887f07ff-91bb-4e62-8e4b-660d5b40b16a")).isEqualTo(TRUNCATED_ENTRY);
+    assertThat(result.get("9d2429cc-155a-4327-8ed7-48fdbe93edd0")).isEqualTo(TRUNCATED_ENTRY);
+  }
+
+  private String readBlob(Blob blob) throws IllegalStateException {
+    try {
+      return new String(blob.getBytes(1, (int) blob.length()), StandardCharsets.UTF_8);
+    } catch (SQLException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+
+  private void insert_entry(String uuid, String content) {
+    db.executeInsert(
+      "CE_SCANNER_CONTEXT",
+      "TASK_UUID", uuid,
+      "CONTEXT_DATA", content.getBytes(StandardCharsets.UTF_8),
+      "CREATED_AT", new Date().getTime(),
+      "UPDATED_AT", new Date().getTime());
+  }
+}

server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v79/TruncateEnvAndSystemVarsFromScannerContextTest/ce_scanner_context.sql

@@ -0,0 +1,8 @@
+CREATE TABLE "CE_SCANNER_CONTEXT" (
+  "TASK_UUID" VARCHAR(40) NOT NULL,
+  "CONTEXT_DATA" BLOB NOT NULL,
+  "CREATED_AT" BIGINT NOT NULL,
+  "UPDATED_AT" BIGINT NOT NULL,
+
+  CONSTRAINT "PK_CE_SCANNER_CONTEXT" PRIMARY KEY ("TASK_UUID")
+);

sonar-scanner-engine/src/main/java/org/sonar/scanner/report/AnalysisContextReportPublisher.java

@@ -27,7 +27,6 @@ import java.nio.file.Files;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
 import java.util.TreeSet;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.CoreProperties;
@@ -36,8 +35,6 @@ import org.sonar.api.batch.fs.internal.AbstractProjectOrModule;
 import org.sonar.api.batch.fs.internal.DefaultInputModule;
 import org.sonar.api.batch.fs.internal.InputModuleHierarchy;
 import org.sonar.api.utils.System2;
-import org.sonar.api.utils.log.Logger;
-import org.sonar.api.utils.log.Loggers;
 import org.sonar.core.platform.PluginInfo;
 import org.sonar.scanner.bootstrap.GlobalServerSettings;
 import org.sonar.scanner.bootstrap.ScannerPluginRepository;
@@ -51,8 +48,6 @@ public class AnalysisContextReportPublisher {
 
   private static final String KEY_VALUE_FORMAT = "  - %s=%s";
 
-  private static final Logger LOG = Loggers.get(AnalysisContextReportPublisher.class);
-
   private static final String ENV_PROP_PREFIX = "env.";
   private static final String SONAR_PROP_PREFIX = "sonar.";
   private static final int MAX_WIDTH = 1000;
@@ -81,10 +76,6 @@ public class AnalysisContextReportPublisher {
     }
     File analysisLog = writer.getFileStructure().analysisLog();
     try (BufferedWriter fileWriter = Files.newBufferedWriter(analysisLog.toPath(), StandardCharsets.UTF_8)) {
-      if (LOG.isDebugEnabled()) {
-        writeEnvVariables(fileWriter);
-        writeSystemProps(fileWriter);
-      }
       writePlugins(fileWriter);
       writeGlobalSettings(fileWriter);
       writeProjectSettings(fileWriter);
@@ -101,31 +92,6 @@ public class AnalysisContextReportPublisher {
     }
   }
 
-  private void writeSystemProps(BufferedWriter fileWriter) throws IOException {
-    fileWriter.write("System properties:\n");
-    Properties sysProps = system.properties();
-    for (String prop : new TreeSet<>(sysProps.stringPropertyNames())) {
-      if (prop.startsWith(SONAR_PROP_PREFIX)) {
-        continue;
-      }
-      fileWriter.append(String.format(KEY_VALUE_FORMAT, prop, sysProps.getProperty(prop))).append('\n');
-    }
-  }
-
-  private void writeEnvVariables(BufferedWriter fileWriter) throws IOException {
-    fileWriter.append("Environment variables:\n");
-    Map<String, String> envVariables = system.envVariables();
-    new TreeSet<>(envVariables.keySet())
-      .forEach(envKey -> {
-        try {
-          String envValue = isSensitiveEnvVariable(envKey) ? "******" : envVariables.get(envKey);
-          fileWriter.append(String.format(KEY_VALUE_FORMAT, envKey, envValue)).append('\n');
-        } catch (IOException e) {
-          throw new IllegalStateException(e);
-        }
-      });
-  }
-
   private void writeGlobalSettings(BufferedWriter fileWriter) throws IOException {
     fileWriter.append("Global server settings:\n");
     Map<String, String> props = globalServerSettings.properties();
@@ -199,10 +165,6 @@ public class AnalysisContextReportPublisher {
     return propKey.startsWith(ENV_PROP_PREFIX) && system.envVariables().containsKey(propKey.substring(ENV_PROP_PREFIX.length()));
   }
 
-  private static boolean isSensitiveEnvVariable(String key) {
-    return key.contains("_TOKEN") || key.contains("_PASSWORD") || key.contains("_SECURED");
-  }
-
   private static boolean isSensitiveProperty(String key) {
     return key.equals(CoreProperties.LOGIN) || key.contains(".password") || key.contains(".secured") || key.contains(".token");
   }

sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisContextReportPublisherTest.java

@@ -151,8 +151,7 @@ public class AnalysisContextReportPublisherTest {
     publisher.init(writer);
 
     List<String> lines = FileUtils.readLines(writer.getFileStructure().analysisLog(), StandardCharsets.UTF_8);
-    assertThat(lines).containsExactly("Environment variables:",
-      "System properties:",
+    assertThat(lines).containsExactly(
       "SonarQube plugins:",
       "Global server settings:",
       "Project server settings:",
@@ -162,67 +161,6 @@ public class AnalysisContextReportPublisherTest {
       "  - sonar.projectKey=foo");
   }
 
-  @Test
-  public void shouldNotDumpSQPropsInSystemProps() throws Exception {
-    logTester.setLevel(LoggerLevel.DEBUG);
-    ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
-    Properties props = new Properties();
-    props.setProperty(COM_FOO, "bar");
-    props.setProperty(SONAR_SKIP, "true");
-    when(system2.properties()).thenReturn(props);
-    DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
-      .setBaseDir(temp.newFolder())
-      .setWorkDir(temp.newFolder())
-      .setProperty("sonar.projectKey", "foo")
-      .setProperty(COM_FOO, "bar")
-      .setProperty(SONAR_SKIP, "true"));
-    when(store.allModules()).thenReturn(singletonList(rootModule));
-    when(hierarchy.root()).thenReturn(rootModule);
-
-    publisher.init(writer);
-
-    List<String> lines = FileUtils.readLines(writer.getFileStructure().analysisLog(), StandardCharsets.UTF_8);
-    assertThat(lines).containsExactly("Environment variables:",
-      "System properties:",
-      "  - com.foo=bar",
-      "SonarQube plugins:",
-      "Global server settings:",
-      "Project server settings:",
-      "Project scanner properties:",
-      "  - sonar.projectKey=foo",
-      "  - sonar.skip=true");
-  }
-
-  @Test
-  public void shouldNotDumpEnvTwice() throws Exception {
-    logTester.setLevel(LoggerLevel.DEBUG);
-    ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
-
-    Map<String, String> env = new HashMap<>();
-    env.put(FOO, "BAR");
-    env.put(BIZ, "BAZ");
-    when(system2.envVariables()).thenReturn(env);
-    DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
-      .setBaseDir(temp.newFolder())
-      .setWorkDir(temp.newFolder())
-      .setProperty("sonar.projectKey", "foo")
-      .setProperty("env." + FOO, "BAR"));
-    when(store.allModules()).thenReturn(singletonList(rootModule));
-    when(hierarchy.root()).thenReturn(rootModule);
-    publisher.init(writer);
-
-    String content = FileUtils.readFileToString(writer.getFileStructure().analysisLog(), StandardCharsets.UTF_8);
-    assertThat(content).containsOnlyOnce(FOO);
-    assertThat(content).containsOnlyOnce(BIZ);
-    assertThat(content).containsSubsequence(BIZ, FOO);
-
-
-    content = FileUtils.readFileToString(writer.getFileStructure().analysisLog(), StandardCharsets.UTF_8);
-    assertThat(content).containsOnlyOnce(FOO);
-    assertThat(content).containsOnlyOnce(BIZ);
-    assertThat(content).doesNotContain("env." + FOO);
-  }
-
   @Test
   public void shouldNotDumpSensitiveModuleProperties() throws Exception {
     ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
@@ -314,8 +252,7 @@ public class AnalysisContextReportPublisherTest {
     publisher.init(writer);
 
     List<String> lines = FileUtils.readLines(writer.getFileStructure().analysisLog(), StandardCharsets.UTF_8);
-    assertThat(lines).containsExactly("Environment variables:",
-      "System properties:",
+    assertThat(lines).containsExactly(
       "SonarQube plugins:",
       "Global server settings:",
       "Project server settings:",