From: Sébastien Lesaint Date: Tue, 16 Jan 2018 10:33:04 +0000 (+0100) Subject: SONAR-10288 move multivalue property parsing to sonar-core X-Git-Tag: 7.5~1770 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=caf4144f98ff834c8fd19ab5d0c5379fa05e32f7;p=sonarqube.git SONAR-10288 move multivalue property parsing to sonar-core --- diff --git a/sonar-core/pom.xml b/sonar-core/pom.xml index be0e9836a67..a0e5e0af6ac 100644 --- a/sonar-core/pom.xml +++ b/sonar-core/pom.xml @@ -27,6 +27,10 @@ commons-codec commons-codec + + org.apache.commons + commons-csv + org.picocontainer picocontainer diff --git a/sonar-core/src/main/java/org/sonar/core/config/MultivalueProperty.java b/sonar-core/src/main/java/org/sonar/core/config/MultivalueProperty.java new file mode 100644 index 00000000000..17451ee0cb6 --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/config/MultivalueProperty.java @@ -0,0 +1,207 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.config; + +import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang.ArrayUtils; + +public class MultivalueProperty { + private MultivalueProperty() { + // prevents instantiation + } + + public static String[] parseAsCsv(String key, String value) { + return parseAsCsv(key, value, Function.identity()); + } + + public static String[] parseAsCsv(String key, String value, Function valueProcessor) { + String cleanValue = MultivalueProperty.trimFieldsAndRemoveEmptyFields(value); + List result = new ArrayList<>(); + try (CSVParser csvParser = CSVFormat.RFC4180 + .withHeader((String) null) + .withIgnoreEmptyLines() + .withIgnoreSurroundingSpaces() + .parse(new StringReader(cleanValue))) { + List records = csvParser.getRecords(); + if (records.isEmpty()) { + return ArrayUtils.EMPTY_STRING_ARRAY; + } + processRecords(result, records, valueProcessor); + return result.toArray(new String[result.size()]); + } catch (IOException e) { + throw new IllegalStateException("Property: '" + key + "' doesn't contain a valid CSV value: '" + value + "'", e); + } + } + + /** + * In most cases we expect a single record.
Having multiple records means the input value was splitted over multiple lines (this is common in Maven). + * For example: + * 
+   *   <sonar.exclusions>
+   *     src/foo,
+   *     src/bar,
+   *     src/biz
+   *   <sonar.exclusions>
+   *
+ * In this case records will be merged to form a single list of items. Last item of a record is appended to first item of next record. + *

+ * This is a very curious case, but we try to preserve line break in the middle of an item: + * 

+   *   <sonar.exclusions>
+   *     a
+   *     b,
+   *     c
+   *   <sonar.exclusions>
+   *
+ * will produce ['a\nb', 'c'] + */ + private static void processRecords(List result, List records, Function valueProcessor) { + for (CSVRecord csvRecord : records) { + Iterator it = csvRecord.iterator(); + if (!result.isEmpty()) { + String next = it.next(); + if (!next.isEmpty()) { + int lastItemIdx = result.size() - 1; + String previous = result.get(lastItemIdx); + if (previous.isEmpty()) { + result.set(lastItemIdx, valueProcessor.apply(next)); + } else { + result.set(lastItemIdx, valueProcessor.apply(previous + "\n" + next)); + } + } + } + it.forEachRemaining(s -> result.add(valueProcessor.apply(s))); + } + } + + /** + * Removes the empty fields from the value of a multi-value property from empty fields, including trimming each field. + *

+ * Quotes can be used to prevent an empty field to be removed (as it is used to preserve empty spaces). + *

    + *
  • {@code "" => ""}
    • + *
  • {@code " " => ""}
    • + *
  • {@code "," => ""}
    • + *
  • {@code ",," => ""}
    • + *
  • {@code ",,," => ""}
    • + *
  • {@code ",a" => "a"}
    • + *
  • {@code "a," => "a"}
    • + *
  • {@code ",a," => "a"}
    • + *
  • {@code "a,,b" => "a,b"}
    • + *
  • {@code "a, ,b" => "a,b"}
    • + *
  • {@code "a,\"\",b" => "a,b"}
    • + *
  • {@code "\"a\",\"b\"" => "\"a\",\"b\""}
    • + *
  • {@code "\" a \",\"b \"" => "\" a \",\"b \""}
    • + *
  • {@code "\"a\",\"\",\"b\"" => "\"a\",\"\",\"b\""}
    • + *
  • {@code "\"a\",\" \",\"b\"" => "\"a\",\" \",\"b\""}
    • + *
  • {@code "\" a,,b,c \",\"d \"" => "\" a,,b,c \",\"d \""}
    • + *
  • {@code "a,\" \",b" => "ab"]}
    • + *
+ */ + @VisibleForTesting + static String trimFieldsAndRemoveEmptyFields(String str) { + char[] chars = str.toCharArray(); + char[] res = new char[chars.length]; + /* + * set when reading the first non trimmable char after a separator char (or the beginning of the string) + * unset when reading a separator + */ + boolean inField = false; + boolean inQuotes = false; + int i = 0; + int resI = 0; + for (; i < chars.length; i++) { + boolean isSeparator = chars[i] == ','; + if (!inQuotes && isSeparator) { + // exiting field (may already be unset) + inField = false; + if (resI > 0) { + resI = retroTrim(res, resI); + } + } else { + boolean isTrimmed = !inQuotes && istrimmable(chars[i]); + if (isTrimmed && !inField) { + // we haven't meet any non trimmable char since the last separator yet + continue; + } + + boolean isEscape = isEscapeChar(chars[i]); + if (isEscape) { + inQuotes = !inQuotes; + } + + // add separator as we already had one field + if (!inField && resI > 0) { + res[resI] = ','; + resI++; + } + + // register in field (may already be set) + inField = true; + // copy current char + res[resI] = chars[i]; + resI++; + } + } + // inQuotes can only be true at this point if quotes are unbalanced + if (!inQuotes) { + // trim end of str + resI = retroTrim(res, resI); + } + return new String(res, 0, resI); + } + + private static boolean isEscapeChar(char aChar) { + return aChar == '"'; + } + + private static boolean istrimmable(char aChar) { + return aChar <= ' '; + } + + /** + * Reads from index {@code resI} to the beginning into {@code res} looking up the location of the trimmable char with + * the lowest index before encountering a non-trimmable char. + *

+ * This basically trims {@code res} from any trimmable char at its end. + * + * @return index of next location to put new char in res + */ + private static int retroTrim(char[] res, int resI) { + int i = resI; + while (i >= 1) { + if (!istrimmable(res[i - 1])) { + return i; + } + i--; + } + return i; + } + +} diff --git a/sonar-core/src/test/java/org/sonar/core/config/MultivaluePropertyTest.java b/sonar-core/src/test/java/org/sonar/core/config/MultivaluePropertyTest.java new file mode 100644 index 00000000000..d9daba88c54 --- /dev/null +++ b/sonar-core/src/test/java/org/sonar/core/config/MultivaluePropertyTest.java @@ -0,0 +1,223 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.core.config; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.Random; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.core.config.MultivalueProperty.trimFieldsAndRemoveEmptyFields; + +@RunWith(DataProviderRunner.class) +public class MultivaluePropertyTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void clean_throws_NPE_if_arg_is_null() { + expectedException.expect(NullPointerException.class); + + trimFieldsAndRemoveEmptyFields(null); + } + + @Test + @UseDataProvider("plains") + public void ignoreEmptyFields(String str) { + assertThat(trimFieldsAndRemoveEmptyFields("")).isEqualTo(""); + assertThat(trimFieldsAndRemoveEmptyFields(str)).isEqualTo(str); + + assertThat(trimFieldsAndRemoveEmptyFields(',' + str)).isEqualTo(str); + assertThat(trimFieldsAndRemoveEmptyFields(str + ',')).isEqualTo(str); + assertThat(trimFieldsAndRemoveEmptyFields(",,," + str)).isEqualTo(str); + assertThat(trimFieldsAndRemoveEmptyFields(str + ",,,")).isEqualTo(str); + + assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str)).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(str + ",,," + str)).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ',' + str)).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields("," + str + ",,," + str)).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(",,," + str + ",,," + str)).isEqualTo(str + ',' + str); + + assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str + ',')).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(str + ",,," + str + ",")).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(str + ",,," + str + ",,")).isEqualTo(str + ',' + str); + + assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ',' + str + ',')).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(",," + str + ',' + str + ',')).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ",," + str + ',')).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ',' + str + ",,")).isEqualTo(str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(",,," + str + ",,," + str + ",,")).isEqualTo(str + ',' + str); + + assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str + ',' + str)).isEqualTo(str + ',' + str + ',' + str); + assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str + ',' + str)).isEqualTo(str + ',' + str + ',' + str); + } + + @DataProvider + public static Object[][] plains() { + return new Object[][] { + {randomAlphanumeric(1)}, + {randomAlphanumeric(2)}, + {randomAlphanumeric(3 + new Random().nextInt(5))} + }; + } + + @Test + @UseDataProvider("emptyAndtrimmable") + public void ignoreEmptyFieldsAndTrimFields(String empty, String trimmable) { + String expected = trimmable.trim(); + assertThat(empty.trim()).isEmpty(); + + assertThat(trimFieldsAndRemoveEmptyFields(trimmable)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + empty)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ",," + empty)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(empty + ',' + trimmable)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(empty + ",," + trimmable)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(empty + ',' + trimmable + ',' + empty)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(empty + ",," + trimmable + ",,," + empty)).isEqualTo(expected); + + assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + empty + ',' + empty)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ",," + empty + ",,," + empty)).isEqualTo(expected); + + assertThat(trimFieldsAndRemoveEmptyFields(empty + ',' + empty + ',' + trimmable)).isEqualTo(expected); + assertThat(trimFieldsAndRemoveEmptyFields(empty + ",,,," + empty + ",," + trimmable)).isEqualTo(expected); + + assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + trimmable)).isEqualTo(expected + ',' + expected); + assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + trimmable + ',' + trimmable)).isEqualTo(expected + ',' + expected + ',' + expected); + assertThat(trimFieldsAndRemoveEmptyFields(trimmable + "," + trimmable + ',' + trimmable)).isEqualTo(expected + ',' + expected + ',' + expected); + } + + @Test + public void trimAccordingToStringTrim() { + String str = randomAlphanumeric(4); + for (int i = 0; i <= ' '; i++) { + String prefixed = (char) i + str; + String suffixed = (char) i + str; + String both = (char) i + str + (char) i; + assertThat(trimFieldsAndRemoveEmptyFields(prefixed)).isEqualTo(prefixed.trim()); + assertThat(trimFieldsAndRemoveEmptyFields(suffixed)).isEqualTo(suffixed.trim()); + assertThat(trimFieldsAndRemoveEmptyFields(both)).isEqualTo(both.trim()); + } + } + + @DataProvider + public static Object[][] emptyAndtrimmable() { + Random random = new Random(); + String oneEmpty = randomTrimmedChars(1, random); + String twoEmpty = randomTrimmedChars(2, random); + String threePlusEmpty = randomTrimmedChars(3 + random.nextInt(5), random); + String onePlusEmpty = randomTrimmedChars(1 + random.nextInt(5), random); + + String plain = randomAlphanumeric(1); + String plainWithtrimmable = randomAlphanumeric(2) + onePlusEmpty + randomAlphanumeric(3); + String quotedWithSeparator = '"' + randomAlphanumeric(3) + ',' + randomAlphanumeric(2) + '"'; + String quotedWithDoubleSeparator = '"' + randomAlphanumeric(3) + ",," + randomAlphanumeric(2) + '"'; + String quotedWithtrimmable = '"' + randomAlphanumeric(3) + onePlusEmpty + randomAlphanumeric(2) + '"'; + + String[] empties = {oneEmpty, twoEmpty, threePlusEmpty}; + String[] strings = {plain, plainWithtrimmable, + onePlusEmpty + plain, plain + onePlusEmpty, onePlusEmpty + plain + onePlusEmpty, + onePlusEmpty + plainWithtrimmable, plainWithtrimmable + onePlusEmpty, onePlusEmpty + plainWithtrimmable + onePlusEmpty, + onePlusEmpty + quotedWithSeparator, quotedWithSeparator + onePlusEmpty, onePlusEmpty + quotedWithSeparator + onePlusEmpty, + onePlusEmpty + quotedWithDoubleSeparator, quotedWithDoubleSeparator + onePlusEmpty, onePlusEmpty + quotedWithDoubleSeparator + onePlusEmpty, + onePlusEmpty + quotedWithtrimmable, quotedWithtrimmable + onePlusEmpty, onePlusEmpty + quotedWithtrimmable + onePlusEmpty + }; + + Object[][] res = new Object[empties.length * strings.length][2]; + int i = 0; + for (String empty : empties) { + for (String string : strings) { + res[i][0] = empty; + res[i][1] = string; + i++; + } + } + return res; + } + + @Test + @UseDataProvider("emptys") + public void quotes_allow_to_preserve_fields(String empty) { + String quotedEmpty = '"' + empty + '"'; + + assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty)).isEqualTo(quotedEmpty); + assertThat(trimFieldsAndRemoveEmptyFields(',' + quotedEmpty)).isEqualTo(quotedEmpty); + assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ',')).isEqualTo(quotedEmpty); + assertThat(trimFieldsAndRemoveEmptyFields(',' + quotedEmpty + ',')).isEqualTo(quotedEmpty); + + assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ',' + quotedEmpty)).isEqualTo(quotedEmpty + ',' + quotedEmpty); + assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ",," + quotedEmpty)).isEqualTo(quotedEmpty + ',' + quotedEmpty); + + assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ',' + quotedEmpty + ',' + quotedEmpty)).isEqualTo(quotedEmpty + ',' + quotedEmpty + ',' + quotedEmpty); + } + + @DataProvider + public static Object[][] emptys() { + Random random = new Random(); + return new Object[][] { + {randomTrimmedChars(1, random)}, + {randomTrimmedChars(2, random)}, + {randomTrimmedChars(3 + random.nextInt(5), random)} + }; + } + + @Test + public void supports_escaped_quote_in_quotes() { + assertThat(trimFieldsAndRemoveEmptyFields("\"f\"\"oo\"")).isEqualTo("\"f\"\"oo\""); + assertThat(trimFieldsAndRemoveEmptyFields("\"f\"\"oo\",\"bar\"\"\"")).isEqualTo("\"f\"\"oo\",\"bar\"\"\""); + } + + @Test + public void does_not_fail_on_unbalanced_quotes() { + assertThat(trimFieldsAndRemoveEmptyFields("\"")).isEqualTo("\""); + assertThat(trimFieldsAndRemoveEmptyFields("\"foo")).isEqualTo("\"foo"); + assertThat(trimFieldsAndRemoveEmptyFields("foo\"")).isEqualTo("foo\""); + + assertThat(trimFieldsAndRemoveEmptyFields("\"foo\",\"")).isEqualTo("\"foo\",\""); + assertThat(trimFieldsAndRemoveEmptyFields("\",\"foo\"")).isEqualTo("\",\"foo\""); + + assertThat(trimFieldsAndRemoveEmptyFields("\"foo\",\", ")).isEqualTo("\"foo\",\", "); + + assertThat(trimFieldsAndRemoveEmptyFields(" a ,,b , c, \"foo\",\" ")).isEqualTo("a,b,c,\"foo\",\" "); + assertThat(trimFieldsAndRemoveEmptyFields("\" a ,,b , c, ")).isEqualTo("\" a ,,b , c, "); + } + + private static final char[] SOME_PRINTABLE_TRIMMABLE_CHARS = { + ' ', '\t', '\n', '\r' + }; + + /** + * Result of randomTrimmedChars being used as arguments to JUnit test method through the DataProvider feature, they + * are printed to surefire report. Some of those chars breaks the parsing of the surefire report during sonar analysis. + * Therefor, we only use a subset of the trimmable chars. + */ + private static String randomTrimmedChars(int length, Random random) { + char[] chars = new char[length]; + for (int i = 0; i < chars.length; i++) { + chars[i] = SOME_PRINTABLE_TRIMMABLE_CHARS[random.nextInt(SOME_PRINTABLE_TRIMMABLE_CHARS.length)]; + } + return new String(chars); + } +} diff --git a/sonar-scanner-engine/pom.xml b/sonar-scanner-engine/pom.xml index 5b2147d285f..622c760e457 100644 --- a/sonar-scanner-engine/pom.xml +++ b/sonar-scanner-engine/pom.xml @@ -76,10 +76,6 @@ com.google.code.gson gson - - org.apache.commons - commons-csv - org.freemarker diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/config/DefaultConfiguration.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/config/DefaultConfiguration.java index 0b32c98a1d5..dfdfa2f9f1d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/config/DefaultConfiguration.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/config/DefaultConfiguration.java @@ -19,19 +19,11 @@ */ package org.sonar.scanner.config; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.concurrent.Immutable; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang.ArrayUtils; import org.sonar.api.config.Configuration; import org.sonar.api.config.Encryption; @@ -44,6 +36,7 @@ import org.sonar.scanner.bootstrap.GlobalAnalysisMode; import static java.util.Objects.requireNonNull; import static org.apache.commons.lang.StringUtils.trim; +import static org.sonar.core.config.MultivalueProperty.parseAsCsv; @Immutable public abstract class DefaultConfiguration implements Configuration { @@ -119,66 +112,6 @@ public abstract class DefaultConfiguration implements Configuration { return ArrayUtils.EMPTY_STRING_ARRAY; } - public static String[] parseAsCsv(String key, String value) { - String cleanValue = MultivaluePropertyCleaner.trimFieldsAndRemoveEmptyFields(value); - List result = new ArrayList<>(); - try (CSVParser csvParser = CSVFormat.RFC4180 - .withHeader((String) null) - .withIgnoreEmptyLines() - .withIgnoreSurroundingSpaces() - .parse(new StringReader(cleanValue))) { - List records = csvParser.getRecords(); - if (records.isEmpty()) { - return ArrayUtils.EMPTY_STRING_ARRAY; - } - processRecords(result, records); - return result.toArray(new String[result.size()]); - } catch (IOException e) { - throw new IllegalStateException("Property: '" + key + "' doesn't contain a valid CSV value: '" + value + "'", e); - } - } - - /** - * In most cases we expect a single record.
Having multiple records means the input value was splitted over multiple lines (this is common in Maven). - * For example: - * 

-   *   <sonar.exclusions>
-   *     src/foo,
-   *     src/bar,
-   *     src/biz
-   *   <sonar.exclusions>
-   *
- * In this case records will be merged to form a single list of items. Last item of a record is appended to first item of next record. - *

- * This is a very curious case, but we try to preserve line break in the middle of an item: - * 

-   *   <sonar.exclusions>
-   *     a
-   *     b,
-   *     c
-   *   <sonar.exclusions>
-   *
- * will produce ['a\nb', 'c'] - */ - private static void processRecords(List result, List records) { - for (CSVRecord csvRecord : records) { - Iterator it = csvRecord.iterator(); - if (!result.isEmpty()) { - String next = it.next(); - if (!next.isEmpty()) { - int lastItemIdx = result.size() - 1; - String previous = result.get(lastItemIdx); - if (previous.isEmpty()) { - result.set(lastItemIdx, next); - } else { - result.set(lastItemIdx, previous + "\n" + next); - } - } - } - it.forEachRemaining(result::add); - } - } - private Optional getInternal(String key) { if (mode.isIssues() && key.endsWith(".secured") && !key.contains(".license")) { throw MessageException.of("Access to the secured property '" + key diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/config/MultivaluePropertyCleaner.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/config/MultivaluePropertyCleaner.java deleted file mode 100644 index a5ae8d807e6..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/config/MultivaluePropertyCleaner.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.config; - -class MultivaluePropertyCleaner { - private MultivaluePropertyCleaner() { - // prevents instantiation - } - - /** - * Removes the empty fields from the value of a multi-value property from empty fields, including trimming each field. - *

- * Quotes can be used to prevent an empty field to be removed (as it is used to preserve empty spaces). - *

    - *
  • {@code "" => ""}
    • - *
  • {@code " " => ""}
    • - *
  • {@code "," => ""}
    • - *
  • {@code ",," => ""}
    • - *
  • {@code ",,," => ""}
    • - *
  • {@code ",a" => "a"}
    • - *
  • {@code "a," => "a"}
    • - *
  • {@code ",a," => "a"}
    • - *
  • {@code "a,,b" => "a,b"}
    • - *
  • {@code "a, ,b" => "a,b"}
    • - *
  • {@code "a,\"\",b" => "a,b"}
    • - *
  • {@code "\"a\",\"b\"" => "\"a\",\"b\""}
    • - *
  • {@code "\" a \",\"b \"" => "\" a \",\"b \""}
    • - *
  • {@code "\"a\",\"\",\"b\"" => "\"a\",\"\",\"b\""}
    • - *
  • {@code "\"a\",\" \",\"b\"" => "\"a\",\" \",\"b\""}
    • - *
  • {@code "\" a,,b,c \",\"d \"" => "\" a,,b,c \",\"d \""}
    • - *
  • {@code "a,\" \",b" => "ab"]}
    • - *
- */ - public static String trimFieldsAndRemoveEmptyFields(String str) { - char[] chars = str.toCharArray(); - char[] res = new char[chars.length]; - /* - * set when reading the first non trimmable char after a separator char (or the beginning of the string) - * unset when reading a separator - */ - boolean inField = false; - boolean inQuotes = false; - int i = 0; - int resI = 0; - for (; i < chars.length; i++) { - boolean isSeparator = chars[i] == ','; - if (!inQuotes && isSeparator) { - // exiting field (may already be unset) - inField = false; - if (resI > 0) { - resI = retroTrim(res, resI); - } - } else { - boolean isTrimmed = !inQuotes && istrimmable(chars[i]); - if (isTrimmed && !inField) { - // we haven't meet any non trimmable char since the last separator yet - continue; - } - - boolean isEscape = isEscapeChar(chars[i]); - if (isEscape) { - inQuotes = !inQuotes; - } - - // add separator as we already had one field - if (!inField && resI > 0) { - res[resI] = ','; - resI++; - } - - // register in field (may already be set) - inField = true; - // copy current char - res[resI] = chars[i]; - resI++; - } - } - // inQuotes can only be true at this point if quotes are unbalanced - if (!inQuotes) { - // trim end of str - resI = retroTrim(res, resI); - } - return new String(res, 0, resI); - } - - private static boolean isEscapeChar(char aChar) { - return aChar == '"'; - } - - private static boolean istrimmable(char aChar) { - return aChar <= ' '; - } - - /** - * Reads from index {@code resI} to the beginning into {@code res} looking up the location of the trimmable char with - * the lowest index before encountering a non-trimmable char. - *

- * This basically trims {@code res} from any trimmable char at its end. - * - * @return index of next location to put new char in res - */ - private static int retroTrim(char[] res, int resI) { - int i = resI; - while (i >= 1) { - if (!istrimmable(res[i - 1])) { - return i; - } - i--; - } - return i; - } - -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorBuilder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorBuilder.java index ac49740357b..9f5716c1a05 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorBuilder.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectReactorBuilder.java @@ -45,9 +45,10 @@ import org.sonar.api.utils.log.Loggers; import org.sonar.api.utils.log.Profiler; import org.sonar.scanner.analysis.AnalysisProperties; import org.sonar.scanner.bootstrap.DroppedPropertyChecker; -import org.sonar.scanner.config.DefaultConfiguration; import org.sonar.scanner.util.ScannerUtils; +import static org.sonar.core.config.MultivalueProperty.parseAsCsv; + /** * Class that creates a project definition based on a set of properties. */ @@ -408,7 +409,7 @@ public class ProjectReactorBuilder { static String[] getListFromProperty(Map properties, String key) { String propValue = properties.get(key); if (propValue != null) { - return DefaultConfiguration.parseAsCsv(key, propValue); + return parseAsCsv(key, propValue); } return new String[0]; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleFileSystemInitializer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleFileSystemInitializer.java index 57dde312c17..12872bb2e49 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleFileSystemInitializer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleFileSystemInitializer.java @@ -38,7 +38,7 @@ import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import static org.sonar.scanner.config.DefaultConfiguration.parseAsCsv; +import static org.sonar.core.config.MultivalueProperty.parseAsCsv; @ScannerSide @Immutable diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/config/MultivaluePropertyCleanerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/config/MultivaluePropertyCleanerTest.java deleted file mode 100644 index 70be5c59b90..00000000000 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/config/MultivaluePropertyCleanerTest.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.config; - -import com.tngtech.java.junit.dataprovider.DataProvider; -import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import com.tngtech.java.junit.dataprovider.UseDataProvider; -import java.util.Random; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; - -import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.scanner.config.MultivaluePropertyCleaner.trimFieldsAndRemoveEmptyFields; - -@RunWith(DataProviderRunner.class) -public class MultivaluePropertyCleanerTest { - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Test - public void clean_throws_NPE_if_arg_is_null() { - expectedException.expect(NullPointerException.class); - - trimFieldsAndRemoveEmptyFields(null); - } - - @Test - @UseDataProvider("plains") - public void ignoreEmptyFields(String str) { - assertThat(trimFieldsAndRemoveEmptyFields("")).isEqualTo(""); - assertThat(trimFieldsAndRemoveEmptyFields(str)).isEqualTo(str); - - assertThat(trimFieldsAndRemoveEmptyFields(',' + str)).isEqualTo(str); - assertThat(trimFieldsAndRemoveEmptyFields(str + ',')).isEqualTo(str); - assertThat(trimFieldsAndRemoveEmptyFields(",,," + str)).isEqualTo(str); - assertThat(trimFieldsAndRemoveEmptyFields(str + ",,,")).isEqualTo(str); - - assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str)).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(str + ",,," + str)).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ',' + str)).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields("," + str + ",,," + str)).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(",,," + str + ",,," + str)).isEqualTo(str + ',' + str); - - assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str + ',')).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(str + ",,," + str + ",")).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(str + ",,," + str + ",,")).isEqualTo(str + ',' + str); - - assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ',' + str + ',')).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(",," + str + ',' + str + ',')).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ",," + str + ',')).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(',' + str + ',' + str + ",,")).isEqualTo(str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(",,," + str + ",,," + str + ",,")).isEqualTo(str + ',' + str); - - assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str + ',' + str)).isEqualTo(str + ',' + str + ',' + str); - assertThat(trimFieldsAndRemoveEmptyFields(str + ',' + str + ',' + str)).isEqualTo(str + ',' + str + ',' + str); - } - - @DataProvider - public static Object[][] plains() { - return new Object[][] { - {randomAlphanumeric(1)}, - {randomAlphanumeric(2)}, - {randomAlphanumeric(3 + new Random().nextInt(5))} - }; - } - - @Test - @UseDataProvider("emptyAndtrimmable") - public void ignoreEmptyFieldsAndTrimFields(String empty, String trimmable) { - String expected = trimmable.trim(); - assertThat(empty.trim()).isEmpty(); - - assertThat(trimFieldsAndRemoveEmptyFields(trimmable)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + empty)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ",," + empty)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(empty + ',' + trimmable)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(empty + ",," + trimmable)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(empty + ',' + trimmable + ',' + empty)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(empty + ",," + trimmable + ",,," + empty)).isEqualTo(expected); - - assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + empty + ',' + empty)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ",," + empty + ",,," + empty)).isEqualTo(expected); - - assertThat(trimFieldsAndRemoveEmptyFields(empty + ',' + empty + ',' + trimmable)).isEqualTo(expected); - assertThat(trimFieldsAndRemoveEmptyFields(empty + ",,,," + empty + ",," + trimmable)).isEqualTo(expected); - - assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + trimmable)).isEqualTo(expected + ',' + expected); - assertThat(trimFieldsAndRemoveEmptyFields(trimmable + ',' + trimmable + ',' + trimmable)).isEqualTo(expected + ',' + expected + ',' + expected); - assertThat(trimFieldsAndRemoveEmptyFields(trimmable + "," + trimmable + ',' + trimmable)).isEqualTo(expected + ',' + expected + ',' + expected); - } - - @Test - public void trimAccordingToStringTrim() { - String str = randomAlphanumeric(4); - for (int i = 0; i <= ' '; i++) { - String prefixed = (char) i + str; - String suffixed = (char) i + str; - String both = (char) i + str + (char) i; - assertThat(trimFieldsAndRemoveEmptyFields(prefixed)).isEqualTo(prefixed.trim()); - assertThat(trimFieldsAndRemoveEmptyFields(suffixed)).isEqualTo(suffixed.trim()); - assertThat(trimFieldsAndRemoveEmptyFields(both)).isEqualTo(both.trim()); - } - } - - @DataProvider - public static Object[][] emptyAndtrimmable() { - Random random = new Random(); - String oneEmpty = randomTrimmedChars(1, random); - String twoEmpty = randomTrimmedChars(2, random); - String threePlusEmpty = randomTrimmedChars(3 + random.nextInt(5), random); - String onePlusEmpty = randomTrimmedChars(1 + random.nextInt(5), random); - - String plain = randomAlphanumeric(1); - String plainWithtrimmable = randomAlphanumeric(2) + onePlusEmpty + randomAlphanumeric(3); - String quotedWithSeparator = '"' + randomAlphanumeric(3) + ',' + randomAlphanumeric(2) + '"'; - String quotedWithDoubleSeparator = '"' + randomAlphanumeric(3) + ",," + randomAlphanumeric(2) + '"'; - String quotedWithtrimmable = '"' + randomAlphanumeric(3) + onePlusEmpty + randomAlphanumeric(2) + '"'; - - String[] empties = {oneEmpty, twoEmpty, threePlusEmpty}; - String[] strings = {plain, plainWithtrimmable, - onePlusEmpty + plain, plain + onePlusEmpty, onePlusEmpty + plain + onePlusEmpty, - onePlusEmpty + plainWithtrimmable, plainWithtrimmable + onePlusEmpty, onePlusEmpty + plainWithtrimmable + onePlusEmpty, - onePlusEmpty + quotedWithSeparator, quotedWithSeparator + onePlusEmpty, onePlusEmpty + quotedWithSeparator + onePlusEmpty, - onePlusEmpty + quotedWithDoubleSeparator, quotedWithDoubleSeparator + onePlusEmpty, onePlusEmpty + quotedWithDoubleSeparator + onePlusEmpty, - onePlusEmpty + quotedWithtrimmable, quotedWithtrimmable + onePlusEmpty, onePlusEmpty + quotedWithtrimmable + onePlusEmpty - }; - - Object[][] res = new Object[empties.length * strings.length][2]; - int i = 0; - for (String empty : empties) { - for (String string : strings) { - res[i][0] = empty; - res[i][1] = string; - i++; - } - } - return res; - } - - @Test - @UseDataProvider("emptys") - public void quotes_allow_to_preserve_fields(String empty) { - String quotedEmpty = '"' + empty + '"'; - - assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty)).isEqualTo(quotedEmpty); - assertThat(trimFieldsAndRemoveEmptyFields(',' + quotedEmpty)).isEqualTo(quotedEmpty); - assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ',')).isEqualTo(quotedEmpty); - assertThat(trimFieldsAndRemoveEmptyFields(',' + quotedEmpty + ',')).isEqualTo(quotedEmpty); - - assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ',' + quotedEmpty)).isEqualTo(quotedEmpty + ',' + quotedEmpty); - assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ",," + quotedEmpty)).isEqualTo(quotedEmpty + ',' + quotedEmpty); - - assertThat(trimFieldsAndRemoveEmptyFields(quotedEmpty + ',' + quotedEmpty + ',' + quotedEmpty)).isEqualTo(quotedEmpty + ',' + quotedEmpty + ',' + quotedEmpty); - } - - @DataProvider - public static Object[][] emptys() { - Random random = new Random(); - return new Object[][] { - {randomTrimmedChars(1, random)}, - {randomTrimmedChars(2, random)}, - {randomTrimmedChars(3 + random.nextInt(5), random)} - }; - } - - @Test - public void supports_escaped_quote_in_quotes() { - assertThat(trimFieldsAndRemoveEmptyFields("\"f\"\"oo\"")).isEqualTo("\"f\"\"oo\""); - assertThat(trimFieldsAndRemoveEmptyFields("\"f\"\"oo\",\"bar\"\"\"")).isEqualTo("\"f\"\"oo\",\"bar\"\"\""); - } - - @Test - public void does_not_fail_on_unbalanced_quotes() { - assertThat(trimFieldsAndRemoveEmptyFields("\"")).isEqualTo("\""); - assertThat(trimFieldsAndRemoveEmptyFields("\"foo")).isEqualTo("\"foo"); - assertThat(trimFieldsAndRemoveEmptyFields("foo\"")).isEqualTo("foo\""); - - assertThat(trimFieldsAndRemoveEmptyFields("\"foo\",\"")).isEqualTo("\"foo\",\""); - assertThat(trimFieldsAndRemoveEmptyFields("\",\"foo\"")).isEqualTo("\",\"foo\""); - - assertThat(trimFieldsAndRemoveEmptyFields("\"foo\",\", ")).isEqualTo("\"foo\",\", "); - - assertThat(trimFieldsAndRemoveEmptyFields(" a ,,b , c, \"foo\",\" ")).isEqualTo("a,b,c,\"foo\",\" "); - assertThat(trimFieldsAndRemoveEmptyFields("\" a ,,b , c, ")).isEqualTo("\" a ,,b , c, "); - } - - private static final char[] SOME_PRINTABLE_TRIMMABLE_CHARS = { - ' ', '\t', '\n', '\r' - }; - - /** - * Result of randomTrimmedChars being used as arguments to JUnit test method through the DataProvider feature, they - * are printed to surefire report. Some of those chars breaks the parsing of the surefire report during sonar analysis. - * Therefor, we only use a subset of the trimmable chars. - */ - private static String randomTrimmedChars(int length, Random random) { - char[] chars = new char[length]; - for (int i = 0; i < chars.length; i++) { - chars[i] = SOME_PRINTABLE_TRIMMABLE_CHARS[random.nextInt(SOME_PRINTABLE_TRIMMABLE_CHARS.length)]; - } - return new String(chars); - } -}