SONAR-6319 SONAR-6321 Feed highlighting and symbols in compute report

15 files changed, 753 insertions, 48 deletions

diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
index 768986d8c0c..4e0af79ccc7 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
@@ -19,6 +19,8 @@
  */
 package org.sonar.api.batch.fs;
 
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+
 import javax.annotation.CheckForNull;
 
 import java.io.File;
@@ -26,6 +28,14 @@ import java.nio.file.Path;
 
 /**
  * This layer over {@link java.io.File} adds information for code analyzers.
+ * For unit testing purpose you can create some {@link DefaultInputFile} and initialize
+ * all fields using 
+ * 
+ * <pre>
+ *   new DefaultInputFile("moduleKey", "relative/path/from/module/baseDir.java")
+ *     .setModuleBaseDir(path)
+ *     .initMetadata(new FileMetadata().readMetadata(someReader));
+ * </pre>
  *
  * @since 4.2
  */
@@ -100,4 +110,20 @@ public interface InputFile extends InputPath {
    */
   int lines();
 
+  /**
+   * Return a {@link TextPointer} in the given file.
+   * @param line Line of the pointer. Start at 1.
+   * @param lineOffset Offset in the line. Start at 0.
+   * @throw {@link IllegalArgumentException} if line or offset is not valid for the given file.
+   */
+  TextPointer newPointer(int line, int lineOffset);
+
+  /**
+   * Return a {@link TextRange} in the given file.
+   * @param start
+   * @param end
+   * @throw {@link IllegalArgumentException} if start or stop pointers are not valid for the given file.
+   */
+  TextRange newRange(TextPointer start, TextPointer end);
+
 }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextPointer.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextPointer.java
new file mode 100644
index 00000000000..63f49b7d407
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextPointer.java
@@ -0,0 +1,39 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs;
+
+/**
+ * Represents a position in a text file {@link InputFile}
+ *
+ * @since 5.2
+ */
+public interface TextPointer extends Comparable<TextPointer> {
+
+  /**
+   * The logical line where this pointer is located. First line is 1.
+   */
+  int line();
+
+  /**
+   * The offset of this pointer in the current line. First position in a line is 0.
+   */
+  int lineOffset();
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextRange.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextRange.java
new file mode 100644
index 00000000000..08239a1e0cd
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextRange.java
@@ -0,0 +1,46 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs;
+
+/**
+ * Represents a text range in an {@link InputFile}
+ *
+ * @since 5.2
+ */
+public interface TextRange {
+
+  /**
+   * Start position of the range
+   */
+  TextPointer start();
+
+  /**
+   * End position of the range
+   */
+  TextPointer end();
+
+  /**
+   * Test if the current range has some common area with another range.
+   * Exemple: say the two ranges are on same line. Range with offsets [1,3] overlaps range with offsets [2,4] but not
+   * range with offset [3,5]
+   */
+  boolean overlap(TextRange another);
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
index a24060a5df1..83b4473fbea 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
@@ -19,7 +19,11 @@
  */
 package org.sonar.api.batch.fs.internal;
 
+import com.google.common.base.Preconditions;
 import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.TextPointer;
+import org.sonar.api.batch.fs.TextRange;
+import org.sonar.api.batch.fs.internal.FileMetadata.Metadata;
 import org.sonar.api.utils.PathUtils;
 
 import javax.annotation.CheckForNull;
@@ -28,6 +32,7 @@ import javax.annotation.Nullable;
 import java.io.File;
 import java.nio.charset.Charset;
 import java.nio.file.Path;
+import java.util.Arrays;
 
 /**
  * @since 4.2
@@ -40,9 +45,13 @@ public class DefaultInputFile implements InputFile {
   private String language;
   private Type type = Type.MAIN;
   private Status status;
-  private int lines;
+  private int lines = -1;
   private Charset charset;
-  private int lastValidOffset;
+  private int lastValidOffset = -1;
+  private String hash;
+  private int nonBlankLines;
+  private int[] originalLineOffsets;
+  private boolean empty;
 
   public DefaultInputFile(String moduleKey, String relativePath) {
     this.moduleKey = moduleKey;
@@ -145,6 +154,7 @@ public class DefaultInputFile implements InputFile {
   }
 
   public int lastValidOffset() {
+    Preconditions.checkState(lastValidOffset >= 0, "InputFile is not properly initialized. Please set 'lastValidOffset' property.");
     return lastValidOffset;
   }
 
@@ -153,6 +163,108 @@ public class DefaultInputFile implements InputFile {
     return this;
   }
 
+  /**
+   * Digest hash of the file.
+   */
+  public String hash() {
+    return hash;
+  }
+
+  public int nonBlankLines() {
+    return nonBlankLines;
+  }
+
+  public int[] originalLineOffsets() {
+    Preconditions.checkState(originalLineOffsets != null, "InputFile is not properly initialized. Please set 'originalLineOffsets' property.");
+    Preconditions.checkState(originalLineOffsets.length == lines, "InputFile is not properly initialized. 'originalLineOffsets' property length should be equal to 'lines'");
+    return originalLineOffsets;
+  }
+
+  public DefaultInputFile setHash(String hash) {
+    this.hash = hash;
+    return this;
+  }
+
+  public DefaultInputFile setNonBlankLines(int nonBlankLines) {
+    this.nonBlankLines = nonBlankLines;
+    return this;
+  }
+
+  public DefaultInputFile setOriginalLineOffsets(int[] originalLineOffsets) {
+    this.originalLineOffsets = originalLineOffsets;
+    return this;
+  }
+
+  public boolean isEmpty() {
+    return this.empty;
+  }
+
+  public DefaultInputFile setEmpty(boolean empty) {
+    this.empty = empty;
+    return this;
+  }
+
+  @Override
+  public TextPointer newPointer(int line, int lineOffset) {
+    DefaultTextPointer textPointer = new DefaultTextPointer(line, lineOffset);
+    checkValid(textPointer, "pointer");
+    return textPointer;
+  }
+
+  private void checkValid(TextPointer pointer, String owner) {
+    Preconditions.checkArgument(pointer.line() >= 1, "%s is not a valid line for a file", pointer.line());
+    Preconditions.checkArgument(pointer.line() <= this.lines, "%s is not a valid line for %s. File %s has %s line(s)", pointer.line(), owner, this, lines);
+    Preconditions.checkArgument(pointer.lineOffset() >= 0, "%s is not a valid line offset for a file", pointer.lineOffset());
+    int lineLength = lineLength(pointer.line());
+    Preconditions.checkArgument(pointer.lineOffset() <= lineLength,
+      "%s is not a valid line offset for %s. File %s has %s character(s) at line %s", pointer.lineOffset(), owner, this, lineLength, pointer.line());
+  }
+
+  private int lineLength(int line) {
+    return lastValidGlobalOffsetForLine(line) - originalLineOffsets()[line - 1];
+  }
+
+  private int lastValidGlobalOffsetForLine(int line) {
+    return line < this.lines ? (originalLineOffsets()[line] - 1) : lastValidOffset();
+  }
+
+  @Override
+  public TextRange newRange(TextPointer start, TextPointer end) {
+    checkValid(start, "start pointer");
+    checkValid(end, "end pointer");
+    Preconditions.checkArgument(start.compareTo(end) < 0, "Start pointer %s should be before end pointer %s", start, end);
+    return new DefaultTextRange(start, end);
+  }
+
+  /**
+   * Create Range from global offsets. Used for backward compatibility with older API.
+   */
+  public TextRange newRange(int startOffset, int endOffset) {
+    return newRange(newPointer(startOffset), newPointer(endOffset));
+  }
+
+  public TextPointer newPointer(int globalOffset) {
+    Preconditions.checkArgument(globalOffset >= 0, "%s is not a valid offset for a file", globalOffset);
+    Preconditions.checkArgument(globalOffset <= lastValidOffset(), "%s is not a valid offset for file %s. Max offset is %s", globalOffset, this, lastValidOffset());
+    int line = findLine(globalOffset);
+    int startLineOffset = originalLineOffsets()[line - 1];
+    return new DefaultTextPointer(line, globalOffset - startLineOffset);
+  }
+
+  private int findLine(int globalOffset) {
+    return Math.abs(Arrays.binarySearch(originalLineOffsets(), globalOffset) + 1);
+  }
+
+  public DefaultInputFile initMetadata(Metadata metadata) {
+    this.setLines(metadata.lines);
+    this.setLastValidOffset(metadata.lastValidOffset);
+    this.setNonBlankLines(metadata.nonBlankLines);
+    this.setHash(metadata.hash);
+    this.setOriginalLineOffsets(metadata.originalLineOffsets);
+    this.setEmpty(metadata.empty);
+    return this;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextPointer.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextPointer.java
new file mode 100644
index 00000000000..572aa4cb838
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextPointer.java
@@ -0,0 +1,74 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs.internal;
+
+import org.sonar.api.batch.fs.TextPointer;
+
+/**
+ * @since 5.2
+ */
+public class DefaultTextPointer implements TextPointer {
+
+  private final int line;
+  private final int lineOffset;
+
+  public DefaultTextPointer(int line, int lineOffset) {
+    this.line = line;
+    this.lineOffset = lineOffset;
+  }
+
+  @Override
+  public int line() {
+    return line;
+  }
+
+  @Override
+  public int lineOffset() {
+    return lineOffset;
+  }
+
+  @Override
+  public String toString() {
+    return "[line=" + line + ", lineOffset=" + lineOffset + "]";
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof DefaultTextPointer)) {
+      return false;
+    }
+    DefaultTextPointer other = (DefaultTextPointer) obj;
+    return other.line == this.line && other.lineOffset == this.lineOffset;
+  }
+
+  @Override
+  public int hashCode() {
+    return 37 * this.line + lineOffset;
+  }
+
+  @Override
+  public int compareTo(TextPointer o) {
+    if (this.line == o.line()) {
+      return Integer.compare(this.lineOffset, o.lineOffset());
+    }
+    return Integer.compare(this.line, o.line());
+  }
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextRange.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextRange.java
new file mode 100644
index 00000000000..b976d1656bc
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextRange.java
@@ -0,0 +1,74 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs.internal;
+
+import org.sonar.api.batch.fs.TextPointer;
+import org.sonar.api.batch.fs.TextRange;
+
+/**
+ * @since 5.2
+ */
+public class DefaultTextRange implements TextRange {
+
+  private final TextPointer start;
+  private final TextPointer end;
+
+  public DefaultTextRange(TextPointer start, TextPointer end) {
+    this.start = start;
+    this.end = end;
+  }
+
+  @Override
+  public TextPointer start() {
+    return start;
+  }
+
+  @Override
+  public TextPointer end() {
+    return end;
+  }
+
+  @Override
+  public boolean overlap(TextRange another) {
+    // [A,B] and [C,D]
+    // B > C && D > A
+    return this.end.compareTo(another.start()) > 0 && another.end().compareTo(this.start) > 0;
+  }
+
+  @Override
+  public String toString() {
+    return "Range[from " + start + " to " + end + "]";
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof DefaultTextRange)) {
+      return false;
+    }
+    DefaultTextRange other = (DefaultTextRange) obj;
+    return start.equals(other.start) && end.equals(other.end);
+  }
+
+  @Override
+  public int hashCode() {
+    return start.hashCode() * 17 + end.hashCode();
+  }
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
new file mode 100644
index 00000000000..bb357542239
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
@@ -0,0 +1,331 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs.internal;
+
+import com.google.common.base.Charsets;
+import com.google.common.primitives.Ints;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.input.BOMInputStream;
+import org.sonar.api.BatchComponent;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Computes hash of files. Ends of Lines are ignored, so files with
+ * same content but different EOL encoding have the same hash.
+ */
+public class FileMetadata implements BatchComponent {
+
+  private static final Logger LOG = Loggers.get(FileMetadata.class);
+
+  private static final char LINE_FEED = '\n';
+  private static final char CARRIAGE_RETURN = '\r';
+
+  private abstract static class CharHandler {
+
+    void handleAll(char c) {
+    }
+
+    void handleIgnoreEoL(char c) {
+    }
+
+    void newLine() {
+    }
+
+    void eof() {
+    }
+  }
+
+  private static class LineCounter extends CharHandler {
+    private boolean empty = true;
+    private int lines = 1;
+    private int nonBlankLines = 0;
+    private boolean blankLine = true;
+    boolean alreadyLoggedInvalidCharacter = false;
+    private final File file;
+    private final Charset encoding;
+
+    LineCounter(File file, Charset encoding) {
+      this.file = file;
+      this.encoding = encoding;
+    }
+
+    @Override
+    void handleAll(char c) {
+      this.empty = false;
+      if (!alreadyLoggedInvalidCharacter && c == '\ufffd') {
+        LOG.warn("Invalid character encountered in file {} at line {} for encoding {}. Please fix file content or configure the encoding to be used using property '{}'.", file,
+          lines, encoding, CoreProperties.ENCODING_PROPERTY);
+        alreadyLoggedInvalidCharacter = true;
+      }
+    }
+
+    @Override
+    void newLine() {
+      lines++;
+      if (!blankLine) {
+        nonBlankLines++;
+      }
+      blankLine = true;
+    }
+
+    @Override
+    void handleIgnoreEoL(char c) {
+      if (!Character.isWhitespace(c)) {
+        blankLine = false;
+      }
+    }
+
+    @Override
+    void eof() {
+      if (!blankLine) {
+        nonBlankLines++;
+      }
+    }
+
+    public int lines() {
+      return lines;
+    }
+
+    public int nonBlankLines() {
+      return nonBlankLines;
+    }
+
+    public boolean isEmpty() {
+      return empty;
+    }
+  }
+
+  private static class FileHashComputer extends CharHandler {
+    private MessageDigest globalMd5Digest = DigestUtils.getMd5Digest();
+    private StringBuilder sb = new StringBuilder();
+
+    @Override
+    void handleIgnoreEoL(char c) {
+      sb.append(c);
+    }
+
+    @Override
+    void newLine() {
+      sb.append(LINE_FEED);
+      globalMd5Digest.update(sb.toString().getBytes(Charsets.UTF_8));
+      sb.setLength(0);
+    }
+
+    @Override
+    void eof() {
+      if (sb.length() > 0) {
+        globalMd5Digest.update(sb.toString().getBytes(Charsets.UTF_8));
+      }
+    }
+
+    @CheckForNull
+    public String getHash() {
+      return Hex.encodeHexString(globalMd5Digest.digest());
+    }
+  }
+
+  private static class LineHashComputer extends CharHandler {
+    private final MessageDigest lineMd5Digest = DigestUtils.getMd5Digest();
+    private final StringBuilder sb = new StringBuilder();
+    private final LineHashConsumer consumer;
+    private int line = 1;
+
+    public LineHashComputer(LineHashConsumer consumer) {
+      this.consumer = consumer;
+    }
+
+    @Override
+    void handleIgnoreEoL(char c) {
+      if (!Character.isWhitespace(c)) {
+        sb.append(c);
+      }
+    }
+
+    @Override
+    void newLine() {
+      consumer.consume(line, sb.length() > 0 ? lineMd5Digest.digest(sb.toString().getBytes(Charsets.UTF_8)) : null);
+      sb.setLength(0);
+      line++;
+    }
+
+    @Override
+    void eof() {
+      consumer.consume(line, sb.length() > 0 ? lineMd5Digest.digest(sb.toString().getBytes(Charsets.UTF_8)) : null);
+    }
+
+  }
+
+  private static class LineOffsetCounter extends CharHandler {
+    private int currentOriginalOffset = 0;
+    private List<Integer> originalLineOffsets = new ArrayList<Integer>();
+    private int lastValidOffset = 0;
+
+    public LineOffsetCounter() {
+      originalLineOffsets.add(0);
+    }
+
+    @Override
+    void handleAll(char c) {
+      currentOriginalOffset++;
+    }
+
+    @Override
+    void newLine() {
+      originalLineOffsets.add(currentOriginalOffset);
+    }
+
+    @Override
+    void eof() {
+      lastValidOffset = currentOriginalOffset;
+    }
+
+    public List<Integer> getOriginalLineOffsets() {
+      return originalLineOffsets;
+    }
+
+    public int getLastValidOffset() {
+      return lastValidOffset;
+    }
+
+  }
+
+  /**
+   * Compute hash of a file ignoring line ends differences.
+   * Maximum performance is needed.
+   */
+  public Metadata readMetadata(File file, Charset encoding) {
+    LineCounter lineCounter = new LineCounter(file, encoding);
+    FileHashComputer fileHashComputer = new FileHashComputer();
+    LineOffsetCounter lineOffsetCounter = new LineOffsetCounter();
+    readFile(file, encoding, lineCounter, fileHashComputer, lineOffsetCounter);
+    return new Metadata(lineCounter.lines(), lineCounter.nonBlankLines(), fileHashComputer.getHash(), lineOffsetCounter.getOriginalLineOffsets(),
+      lineOffsetCounter.getLastValidOffset(),
+      lineCounter.isEmpty());
+  }
+
+  /**
+   * For testing purpose
+   */
+  public Metadata readMetadata(Reader reader) {
+    LineCounter lineCounter = new LineCounter(new File("fromString"), Charsets.UTF_16);
+    FileHashComputer fileHashComputer = new FileHashComputer();
+    LineOffsetCounter lineOffsetCounter = new LineOffsetCounter();
+    try {
+      read(reader, lineCounter, fileHashComputer, lineOffsetCounter);
+    } catch (IOException e) {
+      throw new IllegalStateException("Should never occurs", e);
+    }
+    return new Metadata(lineCounter.lines(), lineCounter.nonBlankLines(), fileHashComputer.getHash(), lineOffsetCounter.getOriginalLineOffsets(),
+      lineOffsetCounter.getLastValidOffset(),
+      lineCounter.isEmpty());
+  }
+
+  private static void readFile(File file, Charset encoding, CharHandler... handlers) {
+    try (BOMInputStream bomIn = new BOMInputStream(new FileInputStream(file),
+      ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
+      Reader reader = new BufferedReader(new InputStreamReader(bomIn, encoding))) {
+      read(reader, handlers);
+    } catch (IOException e) {
+      throw new IllegalStateException(String.format("Fail to read file '%s' with encoding '%s'", file.getAbsolutePath(), encoding), e);
+    }
+  }
+
+  private static void read(Reader reader, CharHandler... handlers) throws IOException {
+    char c = (char) 0;
+    int i = reader.read();
+    boolean afterCR = false;
+    while (i != -1) {
+      c = (char) i;
+      if (afterCR) {
+        for (CharHandler handler : handlers) {
+          if (c != CARRIAGE_RETURN && c != LINE_FEED) {
+            handler.handleIgnoreEoL(c);
+          }
+          handler.handleAll(c);
+          handler.newLine();
+        }
+        afterCR = c == CARRIAGE_RETURN;
+      } else if (c == LINE_FEED) {
+        for (CharHandler handler : handlers) {
+          handler.handleAll(c);
+          handler.newLine();
+        }
+      } else if (c == CARRIAGE_RETURN) {
+        afterCR = true;
+        for (CharHandler handler : handlers) {
+          handler.handleAll(c);
+        }
+      } else {
+        for (CharHandler handler : handlers) {
+          handler.handleIgnoreEoL(c);
+          handler.handleAll(c);
+        }
+      }
+      i = reader.read();
+    }
+    for (CharHandler handler : handlers) {
+      handler.eof();
+    }
+  }
+
+  public static class Metadata {
+    final int lines;
+    final int nonBlankLines;
+    final String hash;
+    final int[] originalLineOffsets;
+    final int lastValidOffset;
+    final boolean empty;
+
+    private Metadata(int lines, int nonBlankLines, String hash, List<Integer> originalLineOffsets, int lastValidOffset, boolean empty) {
+      this.lines = lines;
+      this.nonBlankLines = nonBlankLines;
+      this.hash = hash;
+      this.empty = empty;
+      this.originalLineOffsets = Ints.toArray(originalLineOffsets);
+      this.lastValidOffset = lastValidOffset;
+    }
+  }
+
+  public static interface LineHashConsumer {
+
+    void consume(int lineIdx, @Nullable byte[] hash);
+
+  }
+
+  /**
+   * Compute a MD5 hash of each line of the file after removing of all blank chars
+   */
+  public static void computeLineHashesForIssueTracking(DefaultInputFile f, LineHashConsumer consumer) {
+    readFile(f.file(), f.charset(), new LineHashComputer(consumer));
+  }
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java
index 60e0db9a2d3..50bb7cbab30 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java
@@ -36,7 +36,7 @@ import java.util.Set;
 
 public class DefaultHighlighting extends DefaultStorable implements NewHighlighting {
 
-  private InputFile inputFile;
+  private DefaultInputFile inputFile;
   private Set<SyntaxHighlightingRule> syntaxHighlightingRuleSet;
 
   public DefaultHighlighting() {
@@ -48,9 +48,9 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
     syntaxHighlightingRuleSet = Sets.newTreeSet(new Comparator<SyntaxHighlightingRule>() {
       @Override
       public int compare(SyntaxHighlightingRule left, SyntaxHighlightingRule right) {
-        int result = left.getStartPosition() - right.getStartPosition();
+        int result = left.range().start().compareTo(right.range().start());
         if (result == 0) {
-          result = right.getEndPosition() - left.getEndPosition();
+          result = right.range().end().compareTo(left.range().end());
         }
         return result;
       }
@@ -67,9 +67,9 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
       SyntaxHighlightingRule previous = it.next();
       while (it.hasNext()) {
         SyntaxHighlightingRule current = it.next();
-        if (previous.getEndPosition() > current.getStartPosition() && !(previous.getEndPosition() >= current.getEndPosition())) {
-          String errorMsg = String.format("Cannot register highlighting rule for characters from %s to %s as it " +
-            "overlaps at least one existing rule", current.getStartPosition(), current.getEndPosition());
+        if (previous.range().end().compareTo(current.range().start()) > 0 && !(previous.range().end().compareTo(current.range().end()) >= 0)) {
+          String errorMsg = String.format("Cannot register highlighting rule for characters at %s as it " +
+            "overlaps at least one existing rule", current.range());
           throw new IllegalStateException(errorMsg);
         }
         previous = current;
@@ -80,7 +80,7 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
   @Override
   public DefaultHighlighting onFile(InputFile inputFile) {
     Preconditions.checkNotNull(inputFile, "file can't be null");
-    this.inputFile = inputFile;
+    this.inputFile = (DefaultInputFile) inputFile;
     return this;
   }
 
@@ -91,21 +91,11 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
   @Override
   public DefaultHighlighting highlight(int startOffset, int endOffset, TypeOfText typeOfText) {
     Preconditions.checkState(inputFile != null, "Call onFile() first");
-    int maxValidOffset = ((DefaultInputFile) inputFile).lastValidOffset();
-    checkOffset(startOffset, maxValidOffset, "startOffset");
-    checkOffset(endOffset, maxValidOffset, "endOffset");
-    Preconditions.checkArgument(startOffset < endOffset, "startOffset (" + startOffset + ") should be < endOffset (" + endOffset + ") for file " + inputFile + ".");
-    SyntaxHighlightingRule syntaxHighlightingRule = SyntaxHighlightingRule.create(startOffset, endOffset,
-      typeOfText);
+    SyntaxHighlightingRule syntaxHighlightingRule = SyntaxHighlightingRule.create(inputFile.newRange(startOffset, endOffset), typeOfText);
     this.syntaxHighlightingRuleSet.add(syntaxHighlightingRule);
     return this;
   }
 
-  private void checkOffset(int offset, int maxValidOffset, String label) {
-    Preconditions.checkArgument(offset >= 0 && offset <= maxValidOffset, "Invalid " + label + " " + offset + ". Should be >= 0 and <= " + maxValidOffset
-      + " for file " + inputFile);
-  }
-
   @Override
   protected void doSave() {
     Preconditions.checkState(inputFile != null, "Call onFile() first");
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java
index 9989449ff00..c9b1f000c09 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java
@@ -19,32 +19,27 @@
  */
 package org.sonar.api.batch.sensor.highlighting.internal;
 
+import org.apache.commons.lang.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+import org.sonar.api.batch.fs.TextRange;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 
-import java.io.Serializable;
+public class SyntaxHighlightingRule {
 
-public class SyntaxHighlightingRule implements Serializable {
-
-  private final int startPosition;
-  private final int endPosition;
+  private final TextRange range;
   private final TypeOfText textType;
 
-  private SyntaxHighlightingRule(int startPosition, int endPosition, TypeOfText textType) {
-    this.startPosition = startPosition;
-    this.endPosition = endPosition;
+  private SyntaxHighlightingRule(TextRange range, TypeOfText textType) {
+    this.range = range;
     this.textType = textType;
   }
 
-  public static SyntaxHighlightingRule create(int startPosition, int endPosition, TypeOfText textType) {
-    return new SyntaxHighlightingRule(startPosition, endPosition, textType);
-  }
-
-  public int getStartPosition() {
-    return startPosition;
+  public static SyntaxHighlightingRule create(TextRange range, TypeOfText textType) {
+    return new SyntaxHighlightingRule(range, textType);
   }
 
-  public int getEndPosition() {
-    return endPosition;
+  public TextRange range() {
+    return range;
   }
 
   public TypeOfText getTextType() {
@@ -53,6 +48,6 @@ public class SyntaxHighlightingRule implements Serializable {
 
   @Override
   public String toString() {
-    return "" + startPosition + "," + endPosition + "," + textType.cssClass();
+    return ReflectionToStringBuilder.toString(this, ToStringStyle.SIMPLE_STYLE);
   }
 }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
index 2846e0676fb..278d9ad1741 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
@@ -27,6 +27,7 @@ import org.sonar.api.batch.fs.InputPath;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputDir;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.DefaultTextPointer;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.rule.internal.ActiveRulesBuilder;
 import org.sonar.api.batch.sensor.Sensor;
@@ -55,12 +56,7 @@ import javax.annotation.Nullable;
 
 import java.io.File;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * Utility class to help testing {@link Sensor}.
@@ -179,14 +175,15 @@ public class SensorContextTester implements SensorContext {
     return new DefaultHighlighting(sensorStorage);
   }
 
-  public List<TypeOfText> highlightingTypeFor(String componentKey, int charIndex) {
+  public List<TypeOfText> highlightingTypeAt(String componentKey, int line, int lineOffset) {
     DefaultHighlighting syntaxHighlightingData = sensorStorage.highlightingByComponent.get(componentKey);
     if (syntaxHighlightingData == null) {
       return Collections.emptyList();
     }
     List<TypeOfText> result = new ArrayList<TypeOfText>();
+    DefaultTextPointer location = new DefaultTextPointer(line, lineOffset);
     for (SyntaxHighlightingRule sortedRule : syntaxHighlightingData.getSyntaxHighlightingRuleSet()) {
-      if (sortedRule.getStartPosition() <= charIndex && sortedRule.getEndPosition() > charIndex) {
+      if (sortedRule.range().start().compareTo(location) <= 0 && sortedRule.range().end().compareTo(location) > 0) {
         result.add(sortedRule.getTextType());
       }
     }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java
index 850d3e6cdc7..121ba2a874c 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.api.batch.sensor.internal;
 
+import org.sonar.api.BatchComponent;
 import org.sonar.api.batch.sensor.dependency.Dependency;
 import org.sonar.api.batch.sensor.duplication.Duplication;
 import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
@@ -29,7 +30,7 @@ import org.sonar.api.batch.sensor.measure.Measure;
  * Interface for storing data computed by sensors.
  * @since 5.1
  */
-public interface SensorStorage {
+public interface SensorStorage extends BatchComponent {
 
   void store(Measure measure);
 
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java b/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java
index a5f7773e745..8f16849892e 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java
@@ -22,6 +22,10 @@ package org.sonar.api.component;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.ServerComponent;
 
+/**
+ * @deprecated since 5.2 unused
+ */
+@Deprecated
 public interface Perspectives extends BatchComponent, ServerComponent {
 
   <P extends Perspective> P as(Class<P> perspectiveClass, Component component);
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java b/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java
index d92592678eb..e8555ed4e35 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java
@@ -32,6 +32,7 @@ import org.sonar.api.ServerComponent;
 import org.sonar.api.config.PropertyDefinitions;
 
 import javax.annotation.Nullable;
+
 import java.util.Collection;
 import java.util.List;
 
@@ -172,7 +173,11 @@ public class ComponentContainer implements BatchComponent, ServerComponent {
     if (component instanceof ComponentAdapter) {
       pico.addAdapter((ComponentAdapter) component);
     } else {
-      pico.as(singleton ? Characteristics.CACHE : Characteristics.NO_CACHE).addComponent(key, component);
+      try {
+        pico.as(singleton ? Characteristics.CACHE : Characteristics.NO_CACHE).addComponent(key, component);
+      } catch (Throwable t) {
+        throw new IllegalStateException("Unable to register component " + getName(component), t);
+      }
       declareExtension(null, component);
     }
     return this;
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java
index dd4c7d40883..091b881fabe 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java
@@ -22,12 +22,19 @@ package org.sonar.api.source;
 
 public interface Symbol {
 
+  /**
+   * @deprecated in 5.2 not used.
+   */
+  @Deprecated
   int getDeclarationStartOffset();
 
+  /**
+   * @deprecated in 5.2 not used.
+   */
+  @Deprecated
   int getDeclarationEndOffset();
 
   /**
-   * @since unused
    * @deprecated in 4.3 not used.
    */
   @Deprecated
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java
index 1cc82162047..f205ec41c17 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java
@@ -44,6 +44,10 @@ public interface Symbolizable extends Perspective {
 
     List<Symbol> symbols();
 
+    /**
+     * @deprecated since 5.2 not used
+     */
+    @Deprecated
     List<Integer> references(Symbol symbol);
   }