diff options
8 files changed, 220 insertions, 49 deletions
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java index 7b6b4911191..31e0880a8fe 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultInputFileValueCoder.java @@ -46,6 +46,7 @@ class DefaultInputFileValueCoder implements ValueCoder { value.putString(f.status().name()); putUTFOrNull(value, f.hash()); value.put(f.lines()); + value.put(f.nonBlankLines()); putUTFOrNull(value, f.encoding()); value.put(f.isEmpty()); value.putLongArray(f.originalLineOffsets()); @@ -76,6 +77,7 @@ class DefaultInputFileValueCoder implements ValueCoder { file.setStatus(InputFile.Status.valueOf(value.getString())); file.setHash(value.getString()); file.setLines(value.getInt()); + file.setNonBlankLines(value.getInt()); file.setEncoding(value.getString()); file.setEmpty(value.getBoolean()); file.setOriginalLineOffsets(value.getLongArray()); diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java index d90e026f16b..4769bad4e56 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileMetadata.java @@ -23,6 +23,10 @@ import com.google.common.base.Charsets; import com.google.common.primitives.Longs; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.ByteOrderMark; +import org.apache.commons.io.input.BOMInputStream; + +import javax.annotation.CheckForNull; import java.io.BufferedReader; import java.io.File; @@ -45,68 +49,194 @@ class FileMetadata { private static final char LINE_FEED = '\n'; private static final char CARRIAGE_RETURN = '\r'; - private static final char BOM = '\uFEFF'; + + private abstract class CharHandler { + + void handleAll(char c) { + } + + void handleIgnoreEoL(char c) { + } + + void newLine() { + } + + void eof() { + } + } + + private class LineCounter extends CharHandler { + private boolean empty = true; + private int lines = 1; + private int nonBlankLines = 0; + private boolean blankLine = true; + + @Override + void handleAll(char c) { + this.empty = false; + } + + @Override + void newLine() { + lines++; + if (!blankLine) { + nonBlankLines++; + } + blankLine = true; + } + + @Override + void handleIgnoreEoL(char c) { + if (!Character.isWhitespace(c)) { + blankLine = false; + } + } + + @Override + void eof() { + if (!blankLine) { + nonBlankLines++; + } + } + + public int lines() { + return lines; + } + + public int nonBlankLines() { + return nonBlankLines; + } + + public boolean isEmpty() { + return empty; + } + } + + private class FileHashComputer extends CharHandler { + private MessageDigest globalMd5Digest = DigestUtils.getMd5Digest(); + private boolean emptyFile = true;; + + @Override + void handleIgnoreEoL(char c) { + emptyFile = false; + updateDigestUTF8Char(c, globalMd5Digest); + } + + @Override + void newLine() { + emptyFile = false; + updateDigestUTF8Char(LINE_FEED, globalMd5Digest); + } + + @CheckForNull + public String getHash() { + return emptyFile ? null : Hex.encodeHexString(globalMd5Digest.digest()); + } + } + + private class LineOffsetCounter extends CharHandler { + private long currentOriginalOffset = 0; + private List<Long> originalLineOffsets = new ArrayList<Long>(); + + public LineOffsetCounter() { + originalLineOffsets.add(0L); + } + + @Override + void handleAll(char c) { + currentOriginalOffset++; + } + + @Override + void newLine() { + originalLineOffsets.add(currentOriginalOffset); + } + + public List<Long> getOriginalLineOffsets() { + return originalLineOffsets; + } + + } + + private class LineHashesComputer extends CharHandler { + private List<Object> lineHashes = new ArrayList<Object>(); + private MessageDigest lineMd5Digest = DigestUtils.getMd5Digest(); + private boolean blankLine = true; + + @Override + void handleIgnoreEoL(char c) { + if (!Character.isWhitespace(c)) { + blankLine = false; + updateDigestUTF8Char(c, lineMd5Digest); + } + } + + @Override + void newLine() { + lineHashes.add(blankLine ? null : lineMd5Digest.digest()); + blankLine = true; + } + + @Override + void eof() { + lineHashes.add(blankLine ? null : lineMd5Digest.digest()); + } + + public byte[][] lineHashes() { + return lineHashes.toArray(new byte[0][]); + } + } /** * Compute hash of a file ignoring line ends differences. * Maximum performance is needed. */ Metadata read(File file, Charset encoding) { - long currentOriginalOffset = 0; - List<Long> originalLineOffsets = new ArrayList<Long>(); - List<Object> lineHashes = new ArrayList<Object>(); - int lines = 1; char c = (char) 0; - try (Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding))) { - MessageDigest globalMd5Digest = DigestUtils.getMd5Digest(); - MessageDigest lineMd5Digest = DigestUtils.getMd5Digest(); + LineCounter lineCounter = new LineCounter(); + FileHashComputer fileHashComputer = new FileHashComputer(); + LineOffsetCounter lineOffsetCounter = new LineOffsetCounter(); + LineHashesComputer lineHashesComputer = new LineHashesComputer(); + CharHandler[] handlers = new CharHandler[] {lineCounter, fileHashComputer, lineOffsetCounter, lineHashesComputer}; + try (BOMInputStream bomIn = new BOMInputStream(new FileInputStream(file), + ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE); + Reader reader = new BufferedReader(new InputStreamReader(bomIn, encoding))) { int i = reader.read(); boolean afterCR = false; - // First offset of first line is always 0 - originalLineOffsets.add(0L); - boolean blankline = true; while (i != -1) { c = (char) i; - if (c == BOM) { - // Ignore - i = reader.read(); - continue; - } - currentOriginalOffset++; if (afterCR) { - afterCR = false; - if (c == LINE_FEED) { - originalLineOffsets.set(originalLineOffsets.size() - 1, originalLineOffsets.get(originalLineOffsets.size() - 1) + 1); - // Ignore - i = reader.read(); - continue; + for (CharHandler handler : handlers) { + if (c != CARRIAGE_RETURN && c != LINE_FEED) { + handler.handleIgnoreEoL(c); + } + handler.handleAll(c); + handler.newLine(); } - } - if (c == CARRIAGE_RETURN) { + afterCR = c == CARRIAGE_RETURN; + } else if (c == LINE_FEED) { + for (CharHandler handler : handlers) { + handler.handleAll(c); + handler.newLine(); + } + } else if (c == CARRIAGE_RETURN) { afterCR = true; - c = LINE_FEED; - } - if (c == LINE_FEED) { - lines++; - originalLineOffsets.add(currentOriginalOffset); - lineHashes.add(blankline ? null : lineMd5Digest.digest()); - blankline = true; + for (CharHandler handler : handlers) { + handler.handleAll(c); + } } else { - if (!Character.isWhitespace(c)) { - blankline = false; - updateDigestUTF8Char(c, lineMd5Digest); + for (CharHandler handler : handlers) { + handler.handleIgnoreEoL(c); + handler.handleAll(c); } } - updateDigestUTF8Char(c, globalMd5Digest); i = reader.read(); } - if (c != (char) -1) { - // Last line - lineHashes.add(blankline ? null : lineMd5Digest.digest()); + for (CharHandler handler : handlers) { + handler.eof(); } - boolean empty = lines == 1 && blankline; - String filehash = empty ? null : Hex.encodeHexString(globalMd5Digest.digest()); - return new Metadata(lines, filehash, originalLineOffsets, lineHashes.toArray(new byte[0][]), empty); + return new Metadata(lineCounter.lines(), lineCounter.nonBlankLines(), fileHashComputer.getHash(), lineOffsetCounter.getOriginalLineOffsets(), + lineHashesComputer.lineHashes(), lineCounter.isEmpty()); } catch (IOException e) { throw new IllegalStateException(String.format("Fail to read file '%s' with encoding '%s'", file.getAbsolutePath(), encoding), e); @@ -128,13 +258,15 @@ class FileMetadata { static class Metadata { final int lines; + final int nonBlankLines; final String hash; final long[] originalLineOffsets; final byte[][] lineHashes; final boolean empty; - private Metadata(int lines, String hash, List<Long> originalLineOffsets, byte[][] lineHashes, boolean empty) { + private Metadata(int lines, int nonBlankLines, String hash, List<Long> originalLineOffsets, byte[][] lineHashes, boolean empty) { this.lines = lines; + this.nonBlankLines = nonBlankLines; this.hash = hash; this.empty = empty; this.originalLineOffsets = Longs.toArray(originalLineOffsets); diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java index c97c47cf67d..b5941d83070 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java @@ -109,6 +109,7 @@ class InputFileBuilder { FileMetadata.Metadata metadata = new FileMetadata().read(inputFile.file(), fs.encoding()); inputFile.setLines(metadata.lines); + inputFile.setNonBlankLines(metadata.nonBlankLines); inputFile.setHash(metadata.hash); inputFile.setOriginalLineOffsets(metadata.originalLineOffsets); inputFile.setLineHashes(metadata.lineHashes); diff --git a/sonar-batch/src/main/java/org/sonar/batch/source/LinesSensor.java b/sonar-batch/src/main/java/org/sonar/batch/source/LinesSensor.java index b32630114cb..4829066972a 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/source/LinesSensor.java +++ b/sonar-batch/src/main/java/org/sonar/batch/source/LinesSensor.java @@ -23,6 +23,7 @@ import org.sonar.api.batch.Phase; import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; +import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.sensor.Sensor; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.SensorDescriptor; @@ -52,6 +53,14 @@ public final class LinesSensor implements Sensor { .withValue(f.lines())) .setFromCore() .save(); + if (f.language() == null) { + // As an approximation for files with no language plugin we consider every non blank line as ncloc + ((DefaultMeasure<Integer>) context.<Integer>newMeasure() + .onFile(f) + .forMetric(CoreMetrics.NCLOC) + .withValue(((DefaultInputFile) f).nonBlankLines())) + .save(); + } } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java index 893bc1a132a..811a24870b7 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java @@ -218,7 +218,7 @@ public class FileSystemMediumTest { TaskResult result = tester.newTask() .properties(builder .put("sonar.sources", "src") - .put("sonar.index_all_files", "true") + .put("sonar.import_unknown_files", "true") .build()) .start(); diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java index 6cd0ed34a0e..e1226a9849e 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java @@ -110,7 +110,7 @@ public class MeasuresMediumTest { srcDir.mkdir(); File xooFile = new File(srcDir, "sample.xoo"); - FileUtils.write(xooFile, "Sample xoo\ncontent"); + FileUtils.write(xooFile, "Sample xoo\n\ncontent"); File otherFile = new File(srcDir, "sample.other"); FileUtils.write(otherFile, "Sample other\ncontent\n"); @@ -124,21 +124,26 @@ public class MeasuresMediumTest { .put("sonar.projectVersion", "1.0-SNAPSHOT") .put("sonar.projectDescription", "Description of Foo Project") .put("sonar.sources", "src") - .put("sonar.index_all_files", "true") + .put("sonar.import_unknown_files", "true") .build()) .start(); - // QP + 2 x lines - assertThat(result.measures()).hasSize(3); + // QP + 2 x lines + 1 x ncloc + assertThat(result.measures()).hasSize(4); assertThat(result.measures()).contains(new DefaultMeasure<Integer>() .forMetric(CoreMetrics.LINES) .onFile(new DefaultInputFile("com.foo.project", "src/sample.xoo")) - .withValue(2)); + .withValue(3)); + assertThat(result.measures()).contains(new DefaultMeasure<Integer>() .forMetric(CoreMetrics.LINES) .onFile(new DefaultInputFile("com.foo.project", "src/sample.other")) .withValue(3)); + assertThat(result.measures()).contains(new DefaultMeasure<Integer>() + .forMetric(CoreMetrics.NCLOC) + .onFile(new DefaultInputFile("com.foo.project", "src/sample.other")) + .withValue(2)); } diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java index 7b01635b67e..ad06995c622 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileMetadataTest.java @@ -47,6 +47,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(1); + assertThat(metadata.nonBlankLines).isEqualTo(0); assertThat(metadata.hash).isNull(); assertThat(metadata.originalLineOffsets).containsOnly(0); assertThat(metadata.lineHashes[0]).isNull(); @@ -60,6 +61,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(3); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz")); assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); @@ -75,6 +77,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(4); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("föo\nbàr\n\u1D11Ebaßz\n")); assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18); assertThat(metadata.lineHashes[0]).containsExactly(md5("föo")); @@ -90,6 +93,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_16); assertThat(metadata.lines).isEqualTo(4); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("föo\nbàr\n\u1D11Ebaßz\n")); assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18); assertThat(metadata.lineHashes[0]).containsExactly(md5("föo")); @@ -105,6 +109,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(3); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz")); assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); @@ -119,6 +124,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(4); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz\n")); assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8, 12); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); @@ -134,6 +140,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(4); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz\n")); assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9, 13); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); @@ -149,6 +156,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(4); + assertThat(metadata.nonBlankLines).isEqualTo(2); assertThat(metadata.hash).isEqualTo(md5Hex("foo\n\n\nbar")); assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 5, 6); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); @@ -164,6 +172,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(3); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz")); assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); @@ -178,6 +187,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(4); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("\nfoo\nbar\nbaz")); assertThat(metadata.originalLineOffsets).containsOnly(0, 1, 5, 10); assertThat(metadata.lineHashes[0]).isNull(); @@ -193,6 +203,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(3); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz")); assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); @@ -207,6 +218,7 @@ public class FileMetadataTest { FileMetadata.Metadata metadata = new FileMetadata().read(tempFile, Charsets.UTF_8); assertThat(metadata.lines).isEqualTo(3); + assertThat(metadata.nonBlankLines).isEqualTo(3); assertThat(metadata.hash).isEqualTo(md5Hex(" foo\nb ar\nbaz \t")); assertThat(metadata.lineHashes[0]).containsOnly(md5("foo")); assertThat(metadata.lineHashes[1]).containsOnly(md5("bar")); diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java index 411e3f8389b..ecfd73199af 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java @@ -38,6 +38,7 @@ public class DefaultInputFile implements InputFile, Serializable { private Status status; private String hash; private int lines; + private int nonBlankLines; private String encoding; private long[] originalLineOffsets; private byte[][] lineHashes; @@ -96,6 +97,10 @@ public class DefaultInputFile implements InputFile, Serializable { return lines; } + public int nonBlankLines() { + return nonBlankLines; + } + /** * Component key. */ @@ -154,6 +159,11 @@ public class DefaultInputFile implements InputFile, Serializable { return this; } + public DefaultInputFile setNonBlankLines(int nonBlankLines) { + this.nonBlankLines = nonBlankLines; + return this; + } + public DefaultInputFile setEncoding(String encoding) { this.encoding = encoding; return this; |