SONAR-6319 SONAR-6321 Feed highlighting and symbols in compute report

20 files changed, 1162 insertions, 94 deletions

diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
index 768986d8c0c..4e0af79ccc7 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
@@ -19,6 +19,8 @@
  */
 package org.sonar.api.batch.fs;
 
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+
 import javax.annotation.CheckForNull;
 
 import java.io.File;
@@ -26,6 +28,14 @@ import java.nio.file.Path;
 
 /**
  * This layer over {@link java.io.File} adds information for code analyzers.
+ * For unit testing purpose you can create some {@link DefaultInputFile} and initialize
+ * all fields using 
+ * 
+ * <pre>
+ *   new DefaultInputFile("moduleKey", "relative/path/from/module/baseDir.java")
+ *     .setModuleBaseDir(path)
+ *     .initMetadata(new FileMetadata().readMetadata(someReader));
+ * </pre>
  *
  * @since 4.2
  */
@@ -100,4 +110,20 @@ public interface InputFile extends InputPath {
    */
   int lines();
 
+  /**
+   * Return a {@link TextPointer} in the given file.
+   * @param line Line of the pointer. Start at 1.
+   * @param lineOffset Offset in the line. Start at 0.
+   * @throw {@link IllegalArgumentException} if line or offset is not valid for the given file.
+   */
+  TextPointer newPointer(int line, int lineOffset);
+
+  /**
+   * Return a {@link TextRange} in the given file.
+   * @param start
+   * @param end
+   * @throw {@link IllegalArgumentException} if start or stop pointers are not valid for the given file.
+   */
+  TextRange newRange(TextPointer start, TextPointer end);
+
 }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextPointer.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextPointer.java
new file mode 100644
index 00000000000..63f49b7d407
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextPointer.java
@@ -0,0 +1,39 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs;
+
+/**
+ * Represents a position in a text file {@link InputFile}
+ *
+ * @since 5.2
+ */
+public interface TextPointer extends Comparable<TextPointer> {
+
+  /**
+   * The logical line where this pointer is located. First line is 1.
+   */
+  int line();
+
+  /**
+   * The offset of this pointer in the current line. First position in a line is 0.
+   */
+  int lineOffset();
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextRange.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextRange.java
new file mode 100644
index 00000000000..08239a1e0cd
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/TextRange.java
@@ -0,0 +1,46 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs;
+
+/**
+ * Represents a text range in an {@link InputFile}
+ *
+ * @since 5.2
+ */
+public interface TextRange {
+
+  /**
+   * Start position of the range
+   */
+  TextPointer start();
+
+  /**
+   * End position of the range
+   */
+  TextPointer end();
+
+  /**
+   * Test if the current range has some common area with another range.
+   * Exemple: say the two ranges are on same line. Range with offsets [1,3] overlaps range with offsets [2,4] but not
+   * range with offset [3,5]
+   */
+  boolean overlap(TextRange another);
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
index a24060a5df1..83b4473fbea 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java
@@ -19,7 +19,11 @@
  */
 package org.sonar.api.batch.fs.internal;
 
+import com.google.common.base.Preconditions;
 import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.TextPointer;
+import org.sonar.api.batch.fs.TextRange;
+import org.sonar.api.batch.fs.internal.FileMetadata.Metadata;
 import org.sonar.api.utils.PathUtils;
 
 import javax.annotation.CheckForNull;
@@ -28,6 +32,7 @@ import javax.annotation.Nullable;
 import java.io.File;
 import java.nio.charset.Charset;
 import java.nio.file.Path;
+import java.util.Arrays;
 
 /**
  * @since 4.2
@@ -40,9 +45,13 @@ public class DefaultInputFile implements InputFile {
   private String language;
   private Type type = Type.MAIN;
   private Status status;
-  private int lines;
+  private int lines = -1;
   private Charset charset;
-  private int lastValidOffset;
+  private int lastValidOffset = -1;
+  private String hash;
+  private int nonBlankLines;
+  private int[] originalLineOffsets;
+  private boolean empty;
 
   public DefaultInputFile(String moduleKey, String relativePath) {
     this.moduleKey = moduleKey;
@@ -145,6 +154,7 @@ public class DefaultInputFile implements InputFile {
   }
 
   public int lastValidOffset() {
+    Preconditions.checkState(lastValidOffset >= 0, "InputFile is not properly initialized. Please set 'lastValidOffset' property.");
     return lastValidOffset;
   }
 
@@ -153,6 +163,108 @@ public class DefaultInputFile implements InputFile {
     return this;
   }
 
+  /**
+   * Digest hash of the file.
+   */
+  public String hash() {
+    return hash;
+  }
+
+  public int nonBlankLines() {
+    return nonBlankLines;
+  }
+
+  public int[] originalLineOffsets() {
+    Preconditions.checkState(originalLineOffsets != null, "InputFile is not properly initialized. Please set 'originalLineOffsets' property.");
+    Preconditions.checkState(originalLineOffsets.length == lines, "InputFile is not properly initialized. 'originalLineOffsets' property length should be equal to 'lines'");
+    return originalLineOffsets;
+  }
+
+  public DefaultInputFile setHash(String hash) {
+    this.hash = hash;
+    return this;
+  }
+
+  public DefaultInputFile setNonBlankLines(int nonBlankLines) {
+    this.nonBlankLines = nonBlankLines;
+    return this;
+  }
+
+  public DefaultInputFile setOriginalLineOffsets(int[] originalLineOffsets) {
+    this.originalLineOffsets = originalLineOffsets;
+    return this;
+  }
+
+  public boolean isEmpty() {
+    return this.empty;
+  }
+
+  public DefaultInputFile setEmpty(boolean empty) {
+    this.empty = empty;
+    return this;
+  }
+
+  @Override
+  public TextPointer newPointer(int line, int lineOffset) {
+    DefaultTextPointer textPointer = new DefaultTextPointer(line, lineOffset);
+    checkValid(textPointer, "pointer");
+    return textPointer;
+  }
+
+  private void checkValid(TextPointer pointer, String owner) {
+    Preconditions.checkArgument(pointer.line() >= 1, "%s is not a valid line for a file", pointer.line());
+    Preconditions.checkArgument(pointer.line() <= this.lines, "%s is not a valid line for %s. File %s has %s line(s)", pointer.line(), owner, this, lines);
+    Preconditions.checkArgument(pointer.lineOffset() >= 0, "%s is not a valid line offset for a file", pointer.lineOffset());
+    int lineLength = lineLength(pointer.line());
+    Preconditions.checkArgument(pointer.lineOffset() <= lineLength,
+      "%s is not a valid line offset for %s. File %s has %s character(s) at line %s", pointer.lineOffset(), owner, this, lineLength, pointer.line());
+  }
+
+  private int lineLength(int line) {
+    return lastValidGlobalOffsetForLine(line) - originalLineOffsets()[line - 1];
+  }
+
+  private int lastValidGlobalOffsetForLine(int line) {
+    return line < this.lines ? (originalLineOffsets()[line] - 1) : lastValidOffset();
+  }
+
+  @Override
+  public TextRange newRange(TextPointer start, TextPointer end) {
+    checkValid(start, "start pointer");
+    checkValid(end, "end pointer");
+    Preconditions.checkArgument(start.compareTo(end) < 0, "Start pointer %s should be before end pointer %s", start, end);
+    return new DefaultTextRange(start, end);
+  }
+
+  /**
+   * Create Range from global offsets. Used for backward compatibility with older API.
+   */
+  public TextRange newRange(int startOffset, int endOffset) {
+    return newRange(newPointer(startOffset), newPointer(endOffset));
+  }
+
+  public TextPointer newPointer(int globalOffset) {
+    Preconditions.checkArgument(globalOffset >= 0, "%s is not a valid offset for a file", globalOffset);
+    Preconditions.checkArgument(globalOffset <= lastValidOffset(), "%s is not a valid offset for file %s. Max offset is %s", globalOffset, this, lastValidOffset());
+    int line = findLine(globalOffset);
+    int startLineOffset = originalLineOffsets()[line - 1];
+    return new DefaultTextPointer(line, globalOffset - startLineOffset);
+  }
+
+  private int findLine(int globalOffset) {
+    return Math.abs(Arrays.binarySearch(originalLineOffsets(), globalOffset) + 1);
+  }
+
+  public DefaultInputFile initMetadata(Metadata metadata) {
+    this.setLines(metadata.lines);
+    this.setLastValidOffset(metadata.lastValidOffset);
+    this.setNonBlankLines(metadata.nonBlankLines);
+    this.setHash(metadata.hash);
+    this.setOriginalLineOffsets(metadata.originalLineOffsets);
+    this.setEmpty(metadata.empty);
+    return this;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextPointer.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextPointer.java
new file mode 100644
index 00000000000..572aa4cb838
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextPointer.java
@@ -0,0 +1,74 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs.internal;
+
+import org.sonar.api.batch.fs.TextPointer;
+
+/**
+ * @since 5.2
+ */
+public class DefaultTextPointer implements TextPointer {
+
+  private final int line;
+  private final int lineOffset;
+
+  public DefaultTextPointer(int line, int lineOffset) {
+    this.line = line;
+    this.lineOffset = lineOffset;
+  }
+
+  @Override
+  public int line() {
+    return line;
+  }
+
+  @Override
+  public int lineOffset() {
+    return lineOffset;
+  }
+
+  @Override
+  public String toString() {
+    return "[line=" + line + ", lineOffset=" + lineOffset + "]";
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof DefaultTextPointer)) {
+      return false;
+    }
+    DefaultTextPointer other = (DefaultTextPointer) obj;
+    return other.line == this.line && other.lineOffset == this.lineOffset;
+  }
+
+  @Override
+  public int hashCode() {
+    return 37 * this.line + lineOffset;
+  }
+
+  @Override
+  public int compareTo(TextPointer o) {
+    if (this.line == o.line()) {
+      return Integer.compare(this.lineOffset, o.lineOffset());
+    }
+    return Integer.compare(this.line, o.line());
+  }
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextRange.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextRange.java
new file mode 100644
index 00000000000..b976d1656bc
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultTextRange.java
@@ -0,0 +1,74 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs.internal;
+
+import org.sonar.api.batch.fs.TextPointer;
+import org.sonar.api.batch.fs.TextRange;
+
+/**
+ * @since 5.2
+ */
+public class DefaultTextRange implements TextRange {
+
+  private final TextPointer start;
+  private final TextPointer end;
+
+  public DefaultTextRange(TextPointer start, TextPointer end) {
+    this.start = start;
+    this.end = end;
+  }
+
+  @Override
+  public TextPointer start() {
+    return start;
+  }
+
+  @Override
+  public TextPointer end() {
+    return end;
+  }
+
+  @Override
+  public boolean overlap(TextRange another) {
+    // [A,B] and [C,D]
+    // B > C && D > A
+    return this.end.compareTo(another.start()) > 0 && another.end().compareTo(this.start) > 0;
+  }
+
+  @Override
+  public String toString() {
+    return "Range[from " + start + " to " + end + "]";
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof DefaultTextRange)) {
+      return false;
+    }
+    DefaultTextRange other = (DefaultTextRange) obj;
+    return start.equals(other.start) && end.equals(other.end);
+  }
+
+  @Override
+  public int hashCode() {
+    return start.hashCode() * 17 + end.hashCode();
+  }
+
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
new file mode 100644
index 00000000000..bb357542239
--- /dev/null
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
@@ -0,0 +1,331 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs.internal;
+
+import com.google.common.base.Charsets;
+import com.google.common.primitives.Ints;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.input.BOMInputStream;
+import org.sonar.api.BatchComponent;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Computes hash of files. Ends of Lines are ignored, so files with
+ * same content but different EOL encoding have the same hash.
+ */
+public class FileMetadata implements BatchComponent {
+
+  private static final Logger LOG = Loggers.get(FileMetadata.class);
+
+  private static final char LINE_FEED = '\n';
+  private static final char CARRIAGE_RETURN = '\r';
+
+  private abstract static class CharHandler {
+
+    void handleAll(char c) {
+    }
+
+    void handleIgnoreEoL(char c) {
+    }
+
+    void newLine() {
+    }
+
+    void eof() {
+    }
+  }
+
+  private static class LineCounter extends CharHandler {
+    private boolean empty = true;
+    private int lines = 1;
+    private int nonBlankLines = 0;
+    private boolean blankLine = true;
+    boolean alreadyLoggedInvalidCharacter = false;
+    private final File file;
+    private final Charset encoding;
+
+    LineCounter(File file, Charset encoding) {
+      this.file = file;
+      this.encoding = encoding;
+    }
+
+    @Override
+    void handleAll(char c) {
+      this.empty = false;
+      if (!alreadyLoggedInvalidCharacter && c == '\ufffd') {
+        LOG.warn("Invalid character encountered in file {} at line {} for encoding {}. Please fix file content or configure the encoding to be used using property '{}'.", file,
+          lines, encoding, CoreProperties.ENCODING_PROPERTY);
+        alreadyLoggedInvalidCharacter = true;
+      }
+    }
+
+    @Override
+    void newLine() {
+      lines++;
+      if (!blankLine) {
+        nonBlankLines++;
+      }
+      blankLine = true;
+    }
+
+    @Override
+    void handleIgnoreEoL(char c) {
+      if (!Character.isWhitespace(c)) {
+        blankLine = false;
+      }
+    }
+
+    @Override
+    void eof() {
+      if (!blankLine) {
+        nonBlankLines++;
+      }
+    }
+
+    public int lines() {
+      return lines;
+    }
+
+    public int nonBlankLines() {
+      return nonBlankLines;
+    }
+
+    public boolean isEmpty() {
+      return empty;
+    }
+  }
+
+  private static class FileHashComputer extends CharHandler {
+    private MessageDigest globalMd5Digest = DigestUtils.getMd5Digest();
+    private StringBuilder sb = new StringBuilder();
+
+    @Override
+    void handleIgnoreEoL(char c) {
+      sb.append(c);
+    }
+
+    @Override
+    void newLine() {
+      sb.append(LINE_FEED);
+      globalMd5Digest.update(sb.toString().getBytes(Charsets.UTF_8));
+      sb.setLength(0);
+    }
+
+    @Override
+    void eof() {
+      if (sb.length() > 0) {
+        globalMd5Digest.update(sb.toString().getBytes(Charsets.UTF_8));
+      }
+    }
+
+    @CheckForNull
+    public String getHash() {
+      return Hex.encodeHexString(globalMd5Digest.digest());
+    }
+  }
+
+  private static class LineHashComputer extends CharHandler {
+    private final MessageDigest lineMd5Digest = DigestUtils.getMd5Digest();
+    private final StringBuilder sb = new StringBuilder();
+    private final LineHashConsumer consumer;
+    private int line = 1;
+
+    public LineHashComputer(LineHashConsumer consumer) {
+      this.consumer = consumer;
+    }
+
+    @Override
+    void handleIgnoreEoL(char c) {
+      if (!Character.isWhitespace(c)) {
+        sb.append(c);
+      }
+    }
+
+    @Override
+    void newLine() {
+      consumer.consume(line, sb.length() > 0 ? lineMd5Digest.digest(sb.toString().getBytes(Charsets.UTF_8)) : null);
+      sb.setLength(0);
+      line++;
+    }
+
+    @Override
+    void eof() {
+      consumer.consume(line, sb.length() > 0 ? lineMd5Digest.digest(sb.toString().getBytes(Charsets.UTF_8)) : null);
+    }
+
+  }
+
+  private static class LineOffsetCounter extends CharHandler {
+    private int currentOriginalOffset = 0;
+    private List<Integer> originalLineOffsets = new ArrayList<Integer>();
+    private int lastValidOffset = 0;
+
+    public LineOffsetCounter() {
+      originalLineOffsets.add(0);
+    }
+
+    @Override
+    void handleAll(char c) {
+      currentOriginalOffset++;
+    }
+
+    @Override
+    void newLine() {
+      originalLineOffsets.add(currentOriginalOffset);
+    }
+
+    @Override
+    void eof() {
+      lastValidOffset = currentOriginalOffset;
+    }
+
+    public List<Integer> getOriginalLineOffsets() {
+      return originalLineOffsets;
+    }
+
+    public int getLastValidOffset() {
+      return lastValidOffset;
+    }
+
+  }
+
+  /**
+   * Compute hash of a file ignoring line ends differences.
+   * Maximum performance is needed.
+   */
+  public Metadata readMetadata(File file, Charset encoding) {
+    LineCounter lineCounter = new LineCounter(file, encoding);
+    FileHashComputer fileHashComputer = new FileHashComputer();
+    LineOffsetCounter lineOffsetCounter = new LineOffsetCounter();
+    readFile(file, encoding, lineCounter, fileHashComputer, lineOffsetCounter);
+    return new Metadata(lineCounter.lines(), lineCounter.nonBlankLines(), fileHashComputer.getHash(), lineOffsetCounter.getOriginalLineOffsets(),
+      lineOffsetCounter.getLastValidOffset(),
+      lineCounter.isEmpty());
+  }
+
+  /**
+   * For testing purpose
+   */
+  public Metadata readMetadata(Reader reader) {
+    LineCounter lineCounter = new LineCounter(new File("fromString"), Charsets.UTF_16);
+    FileHashComputer fileHashComputer = new FileHashComputer();
+    LineOffsetCounter lineOffsetCounter = new LineOffsetCounter();
+    try {
+      read(reader, lineCounter, fileHashComputer, lineOffsetCounter);
+    } catch (IOException e) {
+      throw new IllegalStateException("Should never occurs", e);
+    }
+    return new Metadata(lineCounter.lines(), lineCounter.nonBlankLines(), fileHashComputer.getHash(), lineOffsetCounter.getOriginalLineOffsets(),
+      lineOffsetCounter.getLastValidOffset(),
+      lineCounter.isEmpty());
+  }
+
+  private static void readFile(File file, Charset encoding, CharHandler... handlers) {
+    try (BOMInputStream bomIn = new BOMInputStream(new FileInputStream(file),
+      ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
+      Reader reader = new BufferedReader(new InputStreamReader(bomIn, encoding))) {
+      read(reader, handlers);
+    } catch (IOException e) {
+      throw new IllegalStateException(String.format("Fail to read file '%s' with encoding '%s'", file.getAbsolutePath(), encoding), e);
+    }
+  }
+
+  private static void read(Reader reader, CharHandler... handlers) throws IOException {
+    char c = (char) 0;
+    int i = reader.read();
+    boolean afterCR = false;
+    while (i != -1) {
+      c = (char) i;
+      if (afterCR) {
+        for (CharHandler handler : handlers) {
+          if (c != CARRIAGE_RETURN && c != LINE_FEED) {
+            handler.handleIgnoreEoL(c);
+          }
+          handler.handleAll(c);
+          handler.newLine();
+        }
+        afterCR = c == CARRIAGE_RETURN;
+      } else if (c == LINE_FEED) {
+        for (CharHandler handler : handlers) {
+          handler.handleAll(c);
+          handler.newLine();
+        }
+      } else if (c == CARRIAGE_RETURN) {
+        afterCR = true;
+        for (CharHandler handler : handlers) {
+          handler.handleAll(c);
+        }
+      } else {
+        for (CharHandler handler : handlers) {
+          handler.handleIgnoreEoL(c);
+          handler.handleAll(c);
+        }
+      }
+      i = reader.read();
+    }
+    for (CharHandler handler : handlers) {
+      handler.eof();
+    }
+  }
+
+  public static class Metadata {
+    final int lines;
+    final int nonBlankLines;
+    final String hash;
+    final int[] originalLineOffsets;
+    final int lastValidOffset;
+    final boolean empty;
+
+    private Metadata(int lines, int nonBlankLines, String hash, List<Integer> originalLineOffsets, int lastValidOffset, boolean empty) {
+      this.lines = lines;
+      this.nonBlankLines = nonBlankLines;
+      this.hash = hash;
+      this.empty = empty;
+      this.originalLineOffsets = Ints.toArray(originalLineOffsets);
+      this.lastValidOffset = lastValidOffset;
+    }
+  }
+
+  public static interface LineHashConsumer {
+
+    void consume(int lineIdx, @Nullable byte[] hash);
+
+  }
+
+  /**
+   * Compute a MD5 hash of each line of the file after removing of all blank chars
+   */
+  public static void computeLineHashesForIssueTracking(DefaultInputFile f, LineHashConsumer consumer) {
+    readFile(f.file(), f.charset(), new LineHashComputer(consumer));
+  }
+}
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java
index 60e0db9a2d3..50bb7cbab30 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlighting.java
@@ -36,7 +36,7 @@ import java.util.Set;
 
 public class DefaultHighlighting extends DefaultStorable implements NewHighlighting {
 
-  private InputFile inputFile;
+  private DefaultInputFile inputFile;
   private Set<SyntaxHighlightingRule> syntaxHighlightingRuleSet;
 
   public DefaultHighlighting() {
@@ -48,9 +48,9 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
     syntaxHighlightingRuleSet = Sets.newTreeSet(new Comparator<SyntaxHighlightingRule>() {
       @Override
       public int compare(SyntaxHighlightingRule left, SyntaxHighlightingRule right) {
-        int result = left.getStartPosition() - right.getStartPosition();
+        int result = left.range().start().compareTo(right.range().start());
         if (result == 0) {
-          result = right.getEndPosition() - left.getEndPosition();
+          result = right.range().end().compareTo(left.range().end());
         }
         return result;
       }
@@ -67,9 +67,9 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
       SyntaxHighlightingRule previous = it.next();
       while (it.hasNext()) {
         SyntaxHighlightingRule current = it.next();
-        if (previous.getEndPosition() > current.getStartPosition() && !(previous.getEndPosition() >= current.getEndPosition())) {
-          String errorMsg = String.format("Cannot register highlighting rule for characters from %s to %s as it " +
-            "overlaps at least one existing rule", current.getStartPosition(), current.getEndPosition());
+        if (previous.range().end().compareTo(current.range().start()) > 0 && !(previous.range().end().compareTo(current.range().end()) >= 0)) {
+          String errorMsg = String.format("Cannot register highlighting rule for characters at %s as it " +
+            "overlaps at least one existing rule", current.range());
           throw new IllegalStateException(errorMsg);
         }
         previous = current;
@@ -80,7 +80,7 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
   @Override
   public DefaultHighlighting onFile(InputFile inputFile) {
     Preconditions.checkNotNull(inputFile, "file can't be null");
-    this.inputFile = inputFile;
+    this.inputFile = (DefaultInputFile) inputFile;
     return this;
   }
 
@@ -91,21 +91,11 @@ public class DefaultHighlighting extends DefaultStorable implements NewHighlight
   @Override
   public DefaultHighlighting highlight(int startOffset, int endOffset, TypeOfText typeOfText) {
     Preconditions.checkState(inputFile != null, "Call onFile() first");
-    int maxValidOffset = ((DefaultInputFile) inputFile).lastValidOffset();
-    checkOffset(startOffset, maxValidOffset, "startOffset");
-    checkOffset(endOffset, maxValidOffset, "endOffset");
-    Preconditions.checkArgument(startOffset < endOffset, "startOffset (" + startOffset + ") should be < endOffset (" + endOffset + ") for file " + inputFile + ".");
-    SyntaxHighlightingRule syntaxHighlightingRule = SyntaxHighlightingRule.create(startOffset, endOffset,
-      typeOfText);
+    SyntaxHighlightingRule syntaxHighlightingRule = SyntaxHighlightingRule.create(inputFile.newRange(startOffset, endOffset), typeOfText);
     this.syntaxHighlightingRuleSet.add(syntaxHighlightingRule);
     return this;
   }
 
-  private void checkOffset(int offset, int maxValidOffset, String label) {
-    Preconditions.checkArgument(offset >= 0 && offset <= maxValidOffset, "Invalid " + label + " " + offset + ". Should be >= 0 and <= " + maxValidOffset
-      + " for file " + inputFile);
-  }
-
   @Override
   protected void doSave() {
     Preconditions.checkState(inputFile != null, "Call onFile() first");
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java
index 9989449ff00..c9b1f000c09 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/highlighting/internal/SyntaxHighlightingRule.java
@@ -19,32 +19,27 @@
  */
 package org.sonar.api.batch.sensor.highlighting.internal;
 
+import org.apache.commons.lang.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+import org.sonar.api.batch.fs.TextRange;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 
-import java.io.Serializable;
+public class SyntaxHighlightingRule {
 
-public class SyntaxHighlightingRule implements Serializable {
-
-  private final int startPosition;
-  private final int endPosition;
+  private final TextRange range;
   private final TypeOfText textType;
 
-  private SyntaxHighlightingRule(int startPosition, int endPosition, TypeOfText textType) {
-    this.startPosition = startPosition;
-    this.endPosition = endPosition;
+  private SyntaxHighlightingRule(TextRange range, TypeOfText textType) {
+    this.range = range;
     this.textType = textType;
   }
 
-  public static SyntaxHighlightingRule create(int startPosition, int endPosition, TypeOfText textType) {
-    return new SyntaxHighlightingRule(startPosition, endPosition, textType);
-  }
-
-  public int getStartPosition() {
-    return startPosition;
+  public static SyntaxHighlightingRule create(TextRange range, TypeOfText textType) {
+    return new SyntaxHighlightingRule(range, textType);
   }
 
-  public int getEndPosition() {
-    return endPosition;
+  public TextRange range() {
+    return range;
   }
 
   public TypeOfText getTextType() {
@@ -53,6 +48,6 @@ public class SyntaxHighlightingRule implements Serializable {
 
   @Override
   public String toString() {
-    return "" + startPosition + "," + endPosition + "," + textType.cssClass();
+    return ReflectionToStringBuilder.toString(this, ToStringStyle.SIMPLE_STYLE);
   }
 }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
index 2846e0676fb..278d9ad1741 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
@@ -27,6 +27,7 @@ import org.sonar.api.batch.fs.InputPath;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputDir;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.DefaultTextPointer;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.rule.internal.ActiveRulesBuilder;
 import org.sonar.api.batch.sensor.Sensor;
@@ -55,12 +56,7 @@ import javax.annotation.Nullable;
 
 import java.io.File;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * Utility class to help testing {@link Sensor}.
@@ -179,14 +175,15 @@ public class SensorContextTester implements SensorContext {
     return new DefaultHighlighting(sensorStorage);
   }
 
-  public List<TypeOfText> highlightingTypeFor(String componentKey, int charIndex) {
+  public List<TypeOfText> highlightingTypeAt(String componentKey, int line, int lineOffset) {
     DefaultHighlighting syntaxHighlightingData = sensorStorage.highlightingByComponent.get(componentKey);
     if (syntaxHighlightingData == null) {
       return Collections.emptyList();
     }
     List<TypeOfText> result = new ArrayList<TypeOfText>();
+    DefaultTextPointer location = new DefaultTextPointer(line, lineOffset);
     for (SyntaxHighlightingRule sortedRule : syntaxHighlightingData.getSyntaxHighlightingRuleSet()) {
-      if (sortedRule.getStartPosition() <= charIndex && sortedRule.getEndPosition() > charIndex) {
+      if (sortedRule.range().start().compareTo(location) <= 0 && sortedRule.range().end().compareTo(location) > 0) {
         result.add(sortedRule.getTextType());
       }
     }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java
index 850d3e6cdc7..121ba2a874c 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/internal/SensorStorage.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.api.batch.sensor.internal;
 
+import org.sonar.api.BatchComponent;
 import org.sonar.api.batch.sensor.dependency.Dependency;
 import org.sonar.api.batch.sensor.duplication.Duplication;
 import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
@@ -29,7 +30,7 @@ import org.sonar.api.batch.sensor.measure.Measure;
  * Interface for storing data computed by sensors.
  * @since 5.1
  */
-public interface SensorStorage {
+public interface SensorStorage extends BatchComponent {
 
   void store(Measure measure);
 
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java b/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java
index a5f7773e745..8f16849892e 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/component/Perspectives.java
@@ -22,6 +22,10 @@ package org.sonar.api.component;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.ServerComponent;
 
+/**
+ * @deprecated since 5.2 unused
+ */
+@Deprecated
 public interface Perspectives extends BatchComponent, ServerComponent {
 
   <P extends Perspective> P as(Class<P> perspectiveClass, Component component);
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java b/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java
index d92592678eb..e8555ed4e35 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/platform/ComponentContainer.java
@@ -32,6 +32,7 @@ import org.sonar.api.ServerComponent;
 import org.sonar.api.config.PropertyDefinitions;
 
 import javax.annotation.Nullable;
+
 import java.util.Collection;
 import java.util.List;
 
@@ -172,7 +173,11 @@ public class ComponentContainer implements BatchComponent, ServerComponent {
     if (component instanceof ComponentAdapter) {
       pico.addAdapter((ComponentAdapter) component);
     } else {
-      pico.as(singleton ? Characteristics.CACHE : Characteristics.NO_CACHE).addComponent(key, component);
+      try {
+        pico.as(singleton ? Characteristics.CACHE : Characteristics.NO_CACHE).addComponent(key, component);
+      } catch (Throwable t) {
+        throw new IllegalStateException("Unable to register component " + getName(component), t);
+      }
       declareExtension(null, component);
     }
     return this;
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java
index dd4c7d40883..091b881fabe 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbol.java
@@ -22,12 +22,19 @@ package org.sonar.api.source;
 
 public interface Symbol {
 
+  /**
+   * @deprecated in 5.2 not used.
+   */
+  @Deprecated
   int getDeclarationStartOffset();
 
+  /**
+   * @deprecated in 5.2 not used.
+   */
+  @Deprecated
   int getDeclarationEndOffset();
 
   /**
-   * @since unused
    * @deprecated in 4.3 not used.
    */
   @Deprecated
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java
index 1cc82162047..f205ec41c17 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/source/Symbolizable.java
@@ -44,6 +44,10 @@ public interface Symbolizable extends Perspective {
 
     List<Symbol> symbols();
 
+    /**
+     * @deprecated since 5.2 not used
+     */
+    @Deprecated
     List<Integer> references(Symbol symbol);
   }
 
diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java
index edf072dca79..6b8b6af9fea 100644
--- a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java
+++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java
@@ -27,6 +27,7 @@ import org.sonar.api.batch.fs.InputFile;
 import java.io.File;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.fail;
 
 public class DefaultInputFileTest {
 
@@ -73,4 +74,110 @@ public class DefaultInputFileTest {
     DefaultInputFile file = new DefaultInputFile("ABCDE", "src/Foo.php");
     assertThat(file.toString()).isEqualTo("[moduleKey=ABCDE, relative=src/Foo.php, basedir=null]");
   }
+
+  @Test
+  public void checkValidPointer() {
+    DefaultInputFile file = new DefaultInputFile("ABCDE", "src/Foo.php");
+    file.setLines(2);
+    file.setOriginalLineOffsets(new int[] {0, 10});
+    file.setLastValidOffset(15);
+    assertThat(file.newPointer(1, 0).line()).isEqualTo(1);
+    assertThat(file.newPointer(1, 0).lineOffset()).isEqualTo(0);
+    // Don't fail
+    file.newPointer(1, 9);
+    file.newPointer(2, 0);
+    file.newPointer(2, 5);
+
+    try {
+      file.newPointer(0, 1);
+      fail();
+    } catch (Exception e) {
+      assertThat(e).hasMessage("0 is not a valid line for a file");
+    }
+    try {
+      file.newPointer(3, 1);
+      fail();
+    } catch (Exception e) {
+      assertThat(e).hasMessage("3 is not a valid line for pointer. File [moduleKey=ABCDE, relative=src/Foo.php, basedir=null] has 2 line(s)");
+    }
+    try {
+      file.newPointer(1, -1);
+      fail();
+    } catch (Exception e) {
+      assertThat(e).hasMessage("-1 is not a valid line offset for a file");
+    }
+    try {
+      file.newPointer(1, 10);
+      fail();
+    } catch (Exception e) {
+      assertThat(e).hasMessage("10 is not a valid line offset for pointer. File [moduleKey=ABCDE, relative=src/Foo.php, basedir=null] has 9 character(s) at line 1");
+    }
+  }
+
+  @Test
+  public void checkValidPointerUsingGlobalOffset() {
+    DefaultInputFile file = new DefaultInputFile("ABCDE", "src/Foo.php");
+    file.setLines(2);
+    file.setOriginalLineOffsets(new int[] {0, 10});
+    file.setLastValidOffset(15);
+    assertThat(file.newPointer(0).line()).isEqualTo(1);
+    assertThat(file.newPointer(0).lineOffset()).isEqualTo(0);
+
+    assertThat(file.newPointer(9).line()).isEqualTo(1);
+    assertThat(file.newPointer(9).lineOffset()).isEqualTo(9);
+
+    assertThat(file.newPointer(10).line()).isEqualTo(2);
+    assertThat(file.newPointer(10).lineOffset()).isEqualTo(0);
+
+    assertThat(file.newPointer(15).line()).isEqualTo(2);
+    assertThat(file.newPointer(15).lineOffset()).isEqualTo(5);
+
+    try {
+      file.newPointer(-1);
+      fail();
+    } catch (Exception e) {
+      assertThat(e).hasMessage("-1 is not a valid offset for a file");
+    }
+
+    try {
+      file.newPointer(16);
+      fail();
+    } catch (Exception e) {
+      assertThat(e).hasMessage("16 is not a valid offset for file [moduleKey=ABCDE, relative=src/Foo.php, basedir=null]. Max offset is 15");
+    }
+  }
+
+  @Test
+  public void checkValidRange() {
+    DefaultInputFile file = new DefaultInputFile("ABCDE", "src/Foo.php");
+    file.setLines(2);
+    file.setOriginalLineOffsets(new int[] {0, 10});
+    file.setLastValidOffset(15);
+    assertThat(file.newRange(file.newPointer(1, 0), file.newPointer(2, 1)).start().line()).isEqualTo(1);
+    // Don't fail
+    file.newRange(file.newPointer(1, 0), file.newPointer(1, 1));
+    file.newRange(file.newPointer(1, 0), file.newPointer(1, 9));
+    file.newRange(file.newPointer(1, 0), file.newPointer(2, 0));
+    assertThat(file.newRange(file.newPointer(1, 0), file.newPointer(2, 5))).isEqualTo(file.newRange(0, 15));
+
+    try {
+      file.newRange(file.newPointer(1, 0), file.newPointer(1, 0));
+      fail();
+    } catch (Exception e) {
+      assertThat(e).hasMessage("Start pointer [line=1, lineOffset=0] should be before end pointer [line=1, lineOffset=0]");
+    }
+  }
+
+  @Test
+  public void testRangeOverlap() {
+    DefaultInputFile file = new DefaultInputFile("ABCDE", "src/Foo.php");
+    file.setLines(2);
+    file.setOriginalLineOffsets(new int[] {0, 10});
+    file.setLastValidOffset(15);
+    // Don't fail
+    assertThat(file.newRange(file.newPointer(1, 0), file.newPointer(1, 1)).overlap(file.newRange(file.newPointer(1, 0), file.newPointer(1, 1)))).isTrue();
+    assertThat(file.newRange(file.newPointer(1, 0), file.newPointer(1, 1)).overlap(file.newRange(file.newPointer(1, 0), file.newPointer(1, 2)))).isTrue();
+    assertThat(file.newRange(file.newPointer(1, 0), file.newPointer(1, 1)).overlap(file.newRange(file.newPointer(1, 1), file.newPointer(1, 2)))).isFalse();
+    assertThat(file.newRange(file.newPointer(1, 2), file.newPointer(1, 3)).overlap(file.newRange(file.newPointer(1, 0), file.newPointer(1, 2)))).isFalse();
+  }
 }
diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/FileMetadataTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/FileMetadataTest.java
new file mode 100644
index 00000000000..7696c1afcbf
--- /dev/null
+++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/FileMetadataTest.java
@@ -0,0 +1,273 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.api.batch.fs.internal;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.FileUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.FileMetadata.LineHashConsumer;
+import org.sonar.api.utils.log.LogTester;
+import org.sonar.api.utils.log.LoggerLevel;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.nio.charset.Charset;
+
+import static org.apache.commons.codec.digest.DigestUtils.md5Hex;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class FileMetadataTest {
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  @Rule
+  public LogTester logTester = new LogTester();
+
+  @Test
+  public void empty_file() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.touch(tempFile);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(1);
+    assertThat(metadata.nonBlankLines).isEqualTo(0);
+    assertThat(metadata.hash).isNotEmpty();
+    assertThat(metadata.originalLineOffsets).containsOnly(0);
+    assertThat(metadata.lastValidOffset).isEqualTo(0);
+    assertThat(metadata.empty).isTrue();
+  }
+
+  @Test
+  public void windows_without_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\r\nbar\r\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10);
+    assertThat(metadata.lastValidOffset).isEqualTo(13);
+    assertThat(metadata.empty).isFalse();
+  }
+
+  @Test
+  public void read_with_wrong_encoding() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "marker´s\n", Charset.forName("cp1252"));
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(2);
+    assertThat(metadata.hash).isEqualTo(md5Hex("marker\ufffds\n"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 9);
+  }
+
+  @Test
+  public void non_ascii_utf_8() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "föo\r\nbàr\r\n\u1D11Ebaßz\r\n", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("föo\nbàr\n\u1D11Ebaßz\n"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
+  }
+
+  @Test
+  public void non_ascii_utf_16() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "föo\r\nbàr\r\n\u1D11Ebaßz\r\n", Charsets.UTF_16, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_16);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("föo\nbàr\n\u1D11Ebaßz\n"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 5, 10, 18);
+  }
+
+  @Test
+  public void unix_without_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8);
+    assertThat(metadata.lastValidOffset).isEqualTo(11);
+  }
+
+  @Test
+  public void unix_with_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\nbaz\n", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz\n"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8, 12);
+    assertThat(metadata.lastValidOffset).isEqualTo(12);
+  }
+
+  @Test
+  public void mix_of_newlines_with_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\r\nbaz\n", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz\n"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9, 13);
+  }
+
+  @Test
+  public void several_new_lines() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\n\n\nbar", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.nonBlankLines).isEqualTo(2);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\n\n\nbar"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 5, 6);
+  }
+
+  @Test
+  public void mix_of_newlines_without_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\nbar\r\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
+  }
+
+  @Test
+  public void start_with_newline() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "\nfoo\nbar\r\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("\nfoo\nbar\nbaz"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 1, 5, 10);
+  }
+
+  @Test
+  public void start_with_bom() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "\uFEFFfoo\nbar\r\nbaz", Charsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 9);
+  }
+
+  @Test
+  public void ignore_whitespace_when_computing_line_hashes() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, " foo\nb ar\r\nbaz \t", Charsets.UTF_8, true);
+
+    DefaultInputFile f = new DefaultInputFile("foo", tempFile.getName());
+    f.setModuleBaseDir(tempFile.getParentFile().toPath());
+    f.setCharset(Charsets.UTF_8);
+    FileMetadata.computeLineHashesForIssueTracking(f, new LineHashConsumer() {
+
+      @Override
+      public void consume(int lineIdx, @Nullable byte[] hash) {
+        switch (lineIdx) {
+          case 1:
+            assertThat(Hex.encodeHexString(hash)).isEqualTo(md5Hex("foo"));
+            break;
+          case 2:
+            assertThat(Hex.encodeHexString(hash)).isEqualTo(md5Hex("bar"));
+            break;
+          case 3:
+            assertThat(Hex.encodeHexString(hash)).isEqualTo(md5Hex("baz"));
+            break;
+        }
+      }
+    });
+  }
+
+  @Test
+  public void should_throw_if_file_does_not_exist() throws Exception {
+    File tempFolder = temp.newFolder();
+    File file = new File(tempFolder, "doesNotExist.txt");
+
+    thrown.expect(IllegalStateException.class);
+    thrown.expectMessage("Fail to read file '" + file.getAbsolutePath() + "' with encoding 'UTF-8'");
+
+    new FileMetadata().readMetadata(file, Charsets.UTF_8);
+  }
+
+  @Test
+  public void line_feed_is_included_into_hash() throws Exception {
+    File file1 = temp.newFile();
+    FileUtils.write(file1, "foo\nbar\n", Charsets.UTF_8, true);
+
+    // same as file1, except an additional return carriage
+    File file1a = temp.newFile();
+    FileUtils.write(file1a, "foo\r\nbar\n", Charsets.UTF_8, true);
+
+    File file2 = temp.newFile();
+    FileUtils.write(file2, "foo\nbar", Charsets.UTF_8, true);
+
+    String hash1 = new FileMetadata().readMetadata(file1, Charsets.UTF_8).hash;
+    String hash1a = new FileMetadata().readMetadata(file1a, Charsets.UTF_8).hash;
+    String hash2 = new FileMetadata().readMetadata(file2, Charsets.UTF_8).hash;
+    assertThat(hash1).isEqualTo(hash1a);
+    assertThat(hash1).isNotEqualTo(hash2);
+  }
+
+  @Test
+  public void binary_file_with_unmappable_character() throws Exception {
+    File woff = new File(this.getClass().getResource("glyphicons-halflings-regular.woff").toURI());
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(woff, Charsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(135);
+    assertThat(metadata.nonBlankLines).isEqualTo(134);
+    assertThat(metadata.hash).isNotEmpty();
+    assertThat(metadata.empty).isFalse();
+
+    assertThat(logTester.logs(LoggerLevel.WARN).get(0)).contains("Invalid character encountered in file");
+    assertThat(logTester.logs(LoggerLevel.WARN).get(0)).contains(
+      "glyphicons-halflings-regular.woff at line 1 for encoding UTF-8. Please fix file content or configure the encoding to be used using property 'sonar.sourceEncoding'.");
+  }
+
+}
diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlightingTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlightingTest.java
index 4cf1225e27b..79f7db9e06b 100644
--- a/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlightingTest.java
+++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/highlighting/internal/DefaultHighlightingTest.java
@@ -23,7 +23,10 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.sonar.api.batch.fs.TextRange;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.DefaultTextPointer;
+import org.sonar.api.batch.fs.internal.DefaultTextRange;
 import org.sonar.api.batch.sensor.internal.SensorStorage;
 
 import java.util.Collection;
@@ -36,6 +39,11 @@ import static org.sonar.api.batch.sensor.highlighting.TypeOfText.KEYWORD;
 
 public class DefaultHighlightingTest {
 
+  private static final DefaultInputFile INPUT_FILE = new DefaultInputFile("foo", "src/Foo.java")
+    .setLines(2)
+    .setOriginalLineOffsets(new int[] {0, 50})
+    .setLastValidOffset(100);
+
   private Collection<SyntaxHighlightingRule> highlightingRules;
 
   @Rule
@@ -45,7 +53,7 @@ public class DefaultHighlightingTest {
   public void setUpSampleRules() {
 
     DefaultHighlighting highlightingDataBuilder = new DefaultHighlighting()
-      .onFile(new DefaultInputFile("foo", "src/Foo.java").setLastValidOffset(100))
+      .onFile(INPUT_FILE)
       .highlight(0, 10, COMMENT)
       .highlight(10, 12, KEYWORD)
       .highlight(24, 38, KEYWORD)
@@ -61,17 +69,25 @@ public class DefaultHighlightingTest {
     assertThat(highlightingRules).hasSize(6);
   }
 
+  private static TextRange rangeOf(int startLine, int startOffset, int endLine, int endOffset) {
+    return new DefaultTextRange(new DefaultTextPointer(startLine, startOffset), new DefaultTextPointer(endLine, endOffset));
+  }
+
   @Test
   public void should_order_by_start_then_end_offset() throws Exception {
-    assertThat(highlightingRules).extracting("startPosition").containsOnly(0, 10, 12, 24, 24, 42);
-    assertThat(highlightingRules).extracting("endPosition").containsOnly(10, 12, 20, 38, 65, 50);
+    assertThat(highlightingRules).extracting("range", TextRange.class).containsExactly(rangeOf(1, 0, 1, 10),
+      rangeOf(1, 10, 1, 12),
+      rangeOf(1, 12, 1, 20),
+      rangeOf(1, 24, 2, 15),
+      rangeOf(1, 24, 1, 38),
+      rangeOf(1, 42, 2, 0));
     assertThat(highlightingRules).extracting("textType").containsOnly(COMMENT, KEYWORD, COMMENT, KEYWORD, CPP_DOC, KEYWORD);
   }
 
   @Test
   public void should_suport_overlapping() throws Exception {
     new DefaultHighlighting(mock(SensorStorage.class))
-      .onFile(new DefaultInputFile("foo", "src/Foo.java").setLastValidOffset(100))
+      .onFile(INPUT_FILE)
       .highlight(0, 15, KEYWORD)
       .highlight(8, 12, CPP_DOC)
       .save();
@@ -80,49 +96,14 @@ public class DefaultHighlightingTest {
   @Test
   public void should_prevent_boudaries_overlapping() throws Exception {
     throwable.expect(IllegalStateException.class);
-    throwable.expectMessage("Cannot register highlighting rule for characters from 8 to 15 as it overlaps at least one existing rule");
+    throwable
+      .expectMessage("Cannot register highlighting rule for characters at Range[from [line=1, lineOffset=8] to [line=1, lineOffset=15]] as it overlaps at least one existing rule");
 
     new DefaultHighlighting(mock(SensorStorage.class))
-      .onFile(new DefaultInputFile("foo", "src/Foo.java").setLastValidOffset(100))
+      .onFile(INPUT_FILE)
       .highlight(0, 10, KEYWORD)
       .highlight(8, 15, KEYWORD)
       .save();
   }
 
-  @Test
-  public void should_prevent_invalid_offset() throws Exception {
-    throwable.expect(IllegalArgumentException.class);
-    throwable.expectMessage("Invalid endOffset 15. Should be >= 0 and <= 10 for file [moduleKey=foo, relative=src/Foo.java, basedir=null]");
-
-    new DefaultHighlighting(mock(SensorStorage.class))
-      .onFile(new DefaultInputFile("foo", "src/Foo.java").setLastValidOffset(10))
-      .highlight(0, 10, KEYWORD)
-      .highlight(8, 15, KEYWORD)
-      .save();
-  }
-
-  @Test
-  public void positive_offset() throws Exception {
-    throwable.expect(IllegalArgumentException.class);
-    throwable.expectMessage("Invalid startOffset -8. Should be >= 0 and <= 10 for file [moduleKey=foo, relative=src/Foo.java, basedir=null]");
-
-    new DefaultHighlighting(mock(SensorStorage.class))
-      .onFile(new DefaultInputFile("foo", "src/Foo.java").setLastValidOffset(10))
-      .highlight(0, 10, KEYWORD)
-      .highlight(-8, 15, KEYWORD)
-      .save();
-  }
-
-  @Test
-  public void should_prevent_invalid_offset_order() throws Exception {
-    throwable.expect(IllegalArgumentException.class);
-    throwable.expectMessage("startOffset (18) should be < endOffset (15) for file [moduleKey=foo, relative=src/Foo.java, basedir=null].");
-
-    new DefaultHighlighting(mock(SensorStorage.class))
-      .onFile(new DefaultInputFile("foo", "src/Foo.java").setLastValidOffset(20))
-      .highlight(0, 10, KEYWORD)
-      .highlight(18, 15, KEYWORD)
-      .save();
-  }
-
 }
diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java
index 0be267797b7..c1784a24dfc 100644
--- a/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java
+++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java
@@ -26,6 +26,7 @@ import org.junit.rules.TemporaryFolder;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputDir;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.FileMetadata;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.rule.internal.ActiveRulesBuilder;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
@@ -34,6 +35,7 @@ import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.rule.RuleKey;
 
 import java.io.File;
+import java.io.StringReader;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -142,15 +144,15 @@ public class SensorContextTesterTest {
 
   @Test
   public void testHighlighting() {
-    assertThat(tester.highlightingTypeFor("foo:src/Foo.java", 3)).isEmpty();
+    assertThat(tester.highlightingTypeAt("foo:src/Foo.java", 1, 3)).isEmpty();
     tester.newHighlighting()
-      .onFile(new DefaultInputFile("foo", "src/Foo.java").setLastValidOffset(100))
+      .onFile(new DefaultInputFile("foo", "src/Foo.java").initMetadata(new FileMetadata().readMetadata(new StringReader("annot dsf fds foo bar"))))
       .highlight(0, 4, TypeOfText.ANNOTATION)
       .highlight(8, 10, TypeOfText.CONSTANT)
       .highlight(9, 10, TypeOfText.COMMENT)
       .save();
-    assertThat(tester.highlightingTypeFor("foo:src/Foo.java", 3)).containsExactly(TypeOfText.ANNOTATION);
-    assertThat(tester.highlightingTypeFor("foo:src/Foo.java", 9)).containsExactly(TypeOfText.CONSTANT, TypeOfText.COMMENT);
+    assertThat(tester.highlightingTypeAt("foo:src/Foo.java", 1, 3)).containsExactly(TypeOfText.ANNOTATION);
+    assertThat(tester.highlightingTypeAt("foo:src/Foo.java", 1, 9)).containsExactly(TypeOfText.CONSTANT, TypeOfText.COMMENT);
   }
 
   @Test
diff --git a/sonar-plugin-api/src/test/resources/org/sonar/api/batch/fs/internal/glyphicons-halflings-regular.woff b/sonar-plugin-api/src/test/resources/org/sonar/api/batch/fs/internal/glyphicons-halflings-regular.woff
new file mode 100644
index 00000000000..2cc3e4852a5
--- /dev/null
+++ b/sonar-plugin-api/src/test/resources/org/sonar/api/batch/fs/internal/glyphicons-halflings-regular.woff