Move git diff codes from models to services/gitdiff (#7889)

* move git diff codes from models to services/gitdiff

* fix template

* fix test

* fix template

4 files changed, 1110 insertions, 0 deletions

diff --git a/services/comments/comments.go b/services/comments/comments.go
new file mode 100644
index 0000000000..bd261ff0a5
--- /dev/null
+++ b/services/comments/comments.go
@@ -0,0 +1,69 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package comments
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/gitdiff"
+)
+
+// CreateCodeComment creates a plain code comment at the specified line / path
+func CreateCodeComment(doer *models.User, repo *models.Repository, issue *models.Issue, content, treePath string, line, reviewID int64) (*models.Comment, error) {
+	var commitID, patch string
+	pr, err := models.GetPullRequestByIssueID(issue.ID)
+	if err != nil {
+		return nil, fmt.Errorf("GetPullRequestByIssueID: %v", err)
+	}
+	if err := pr.GetBaseRepo(); err != nil {
+		return nil, fmt.Errorf("GetHeadRepo: %v", err)
+	}
+	gitRepo, err := git.OpenRepository(pr.BaseRepo.RepoPath())
+	if err != nil {
+		return nil, fmt.Errorf("OpenRepository: %v", err)
+	}
+
+	// FIXME validate treePath
+	// Get latest commit referencing the commented line
+	// No need for get commit for base branch changes
+	if line > 0 {
+		commit, err := gitRepo.LineBlame(pr.GetGitRefName(), gitRepo.Path, treePath, uint(line))
+		if err == nil {
+			commitID = commit.ID.String()
+		} else if !strings.Contains(err.Error(), "exit status 128 - fatal: no such path") {
+			return nil, fmt.Errorf("LineBlame[%s, %s, %s, %d]: %v", pr.GetGitRefName(), gitRepo.Path, treePath, line, err)
+		}
+	}
+
+	// Only fetch diff if comment is review comment
+	if reviewID != 0 {
+		headCommitID, err := gitRepo.GetRefCommitID(pr.GetGitRefName())
+		if err != nil {
+			return nil, fmt.Errorf("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err)
+		}
+		patchBuf := new(bytes.Buffer)
+		if err := gitdiff.GetRawDiffForFile(gitRepo.Path, pr.MergeBase, headCommitID, gitdiff.RawDiffNormal, treePath, patchBuf); err != nil {
+			return nil, fmt.Errorf("GetRawDiffForLine[%s, %s, %s, %s]: %v", err, gitRepo.Path, pr.MergeBase, headCommitID, treePath)
+		}
+		patch = gitdiff.CutDiffAroundLine(patchBuf, int64((&models.Comment{Line: line}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines)
+	}
+	return models.CreateComment(&models.CreateCommentOptions{
+		Type:      models.CommentTypeCode,
+		Doer:      doer,
+		Repo:      repo,
+		Issue:     issue,
+		Content:   content,
+		LineNum:   line,
+		TreePath:  treePath,
+		CommitSHA: commitID,
+		ReviewID:  reviewID,
+		Patch:     patch,
+	})
+}
diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go
new file mode 100644
index 0000000000..c2c5675d9f
--- /dev/null
+++ b/services/gitdiff/gitdiff.go
@@ -0,0 +1,828 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package gitdiff
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"html"
+	"html/template"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/charset"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/highlight"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/sergi/go-diff/diffmatchpatch"
+	"github.com/unknwon/com"
+	stdcharset "golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+// DiffLineType represents the type of a DiffLine.
+type DiffLineType uint8
+
+// DiffLineType possible values.
+const (
+	DiffLinePlain DiffLineType = iota + 1
+	DiffLineAdd
+	DiffLineDel
+	DiffLineSection
+)
+
+// DiffFileType represents the type of a DiffFile.
+type DiffFileType uint8
+
+// DiffFileType possible values.
+const (
+	DiffFileAdd DiffFileType = iota + 1
+	DiffFileChange
+	DiffFileDel
+	DiffFileRename
+)
+
+// DiffLine represents a line difference in a DiffSection.
+type DiffLine struct {
+	LeftIdx  int
+	RightIdx int
+	Type     DiffLineType
+	Content  string
+	Comments []*models.Comment
+}
+
+// GetType returns the type of a DiffLine.
+func (d *DiffLine) GetType() int {
+	return int(d.Type)
+}
+
+// CanComment returns whether or not a line can get commented
+func (d *DiffLine) CanComment() bool {
+	return len(d.Comments) == 0 && d.Type != DiffLineSection
+}
+
+// GetCommentSide returns the comment side of the first comment, if not set returns empty string
+func (d *DiffLine) GetCommentSide() string {
+	if len(d.Comments) == 0 {
+		return ""
+	}
+	return d.Comments[0].DiffSide()
+}
+
+// GetLineTypeMarker returns the line type marker
+func (d *DiffLine) GetLineTypeMarker() string {
+	if strings.IndexByte(" +-", d.Content[0]) > -1 {
+		return d.Content[0:1]
+	}
+	return ""
+}
+
+// escape a line's content or return <br> needed for copy/paste purposes
+func getLineContent(content string) string {
+	if len(content) > 0 {
+		return html.EscapeString(content)
+	}
+	return "<br>"
+}
+
+// DiffSection represents a section of a DiffFile.
+type DiffSection struct {
+	Name  string
+	Lines []*DiffLine
+}
+
+var (
+	addedCodePrefix   = []byte(`<span class="added-code">`)
+	removedCodePrefix = []byte(`<span class="removed-code">`)
+	codeTagSuffix     = []byte(`</span>`)
+)
+
+func diffToHTML(diffs []diffmatchpatch.Diff, lineType DiffLineType) template.HTML {
+	buf := bytes.NewBuffer(nil)
+
+	for i := range diffs {
+		switch {
+		case diffs[i].Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd:
+			buf.Write(addedCodePrefix)
+			buf.WriteString(getLineContent(diffs[i].Text))
+			buf.Write(codeTagSuffix)
+		case diffs[i].Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel:
+			buf.Write(removedCodePrefix)
+			buf.WriteString(getLineContent(diffs[i].Text))
+			buf.Write(codeTagSuffix)
+		case diffs[i].Type == diffmatchpatch.DiffEqual:
+			buf.WriteString(getLineContent(diffs[i].Text))
+		}
+	}
+
+	return template.HTML(buf.Bytes())
+}
+
+// GetLine gets a specific line by type (add or del) and file line number
+func (diffSection *DiffSection) GetLine(lineType DiffLineType, idx int) *DiffLine {
+	var (
+		difference    = 0
+		addCount      = 0
+		delCount      = 0
+		matchDiffLine *DiffLine
+	)
+
+LOOP:
+	for _, diffLine := range diffSection.Lines {
+		switch diffLine.Type {
+		case DiffLineAdd:
+			addCount++
+		case DiffLineDel:
+			delCount++
+		default:
+			if matchDiffLine != nil {
+				break LOOP
+			}
+			difference = diffLine.RightIdx - diffLine.LeftIdx
+			addCount = 0
+			delCount = 0
+		}
+
+		switch lineType {
+		case DiffLineDel:
+			if diffLine.RightIdx == 0 && diffLine.LeftIdx == idx-difference {
+				matchDiffLine = diffLine
+			}
+		case DiffLineAdd:
+			if diffLine.LeftIdx == 0 && diffLine.RightIdx == idx+difference {
+				matchDiffLine = diffLine
+			}
+		}
+	}
+
+	if addCount == delCount {
+		return matchDiffLine
+	}
+	return nil
+}
+
+var diffMatchPatch = diffmatchpatch.New()
+
+func init() {
+	diffMatchPatch.DiffEditCost = 100
+}
+
+// GetComputedInlineDiffFor computes inline diff for the given line.
+func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) template.HTML {
+	if setting.Git.DisableDiffHighlight {
+		return template.HTML(getLineContent(diffLine.Content[1:]))
+	}
+	var (
+		compareDiffLine *DiffLine
+		diff1           string
+		diff2           string
+	)
+
+	// try to find equivalent diff line. ignore, otherwise
+	switch diffLine.Type {
+	case DiffLineAdd:
+		compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx)
+		if compareDiffLine == nil {
+			return template.HTML(getLineContent(diffLine.Content[1:]))
+		}
+		diff1 = compareDiffLine.Content
+		diff2 = diffLine.Content
+	case DiffLineDel:
+		compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx)
+		if compareDiffLine == nil {
+			return template.HTML(getLineContent(diffLine.Content[1:]))
+		}
+		diff1 = diffLine.Content
+		diff2 = compareDiffLine.Content
+	default:
+		if strings.IndexByte(" +-", diffLine.Content[0]) > -1 {
+			return template.HTML(getLineContent(diffLine.Content[1:]))
+		}
+		return template.HTML(getLineContent(diffLine.Content))
+	}
+
+	diffRecord := diffMatchPatch.DiffMain(diff1[1:], diff2[1:], true)
+	diffRecord = diffMatchPatch.DiffCleanupEfficiency(diffRecord)
+
+	return diffToHTML(diffRecord, diffLine.Type)
+}
+
+// DiffFile represents a file diff.
+type DiffFile struct {
+	Name               string
+	OldName            string
+	Index              int
+	Addition, Deletion int
+	Type               DiffFileType
+	IsCreated          bool
+	IsDeleted          bool
+	IsBin              bool
+	IsLFSFile          bool
+	IsRenamed          bool
+	IsSubmodule        bool
+	Sections           []*DiffSection
+	IsIncomplete       bool
+}
+
+// GetType returns type of diff file.
+func (diffFile *DiffFile) GetType() int {
+	return int(diffFile.Type)
+}
+
+// GetHighlightClass returns highlight class for a filename.
+func (diffFile *DiffFile) GetHighlightClass() string {
+	return highlight.FileNameToHighlightClass(diffFile.Name)
+}
+
+// Diff represents a difference between two git trees.
+type Diff struct {
+	TotalAddition, TotalDeletion int
+	Files                        []*DiffFile
+	IsIncomplete                 bool
+}
+
+// LoadComments loads comments into each line
+func (diff *Diff) LoadComments(issue *models.Issue, currentUser *models.User) error {
+	allComments, err := models.FetchCodeComments(issue, currentUser)
+	if err != nil {
+		return err
+	}
+	for _, file := range diff.Files {
+		if lineCommits, ok := allComments[file.Name]; ok {
+			for _, section := range file.Sections {
+				for _, line := range section.Lines {
+					if comments, ok := lineCommits[int64(line.LeftIdx*-1)]; ok {
+						line.Comments = append(line.Comments, comments...)
+					}
+					if comments, ok := lineCommits[int64(line.RightIdx)]; ok {
+						line.Comments = append(line.Comments, comments...)
+					}
+					sort.SliceStable(line.Comments, func(i, j int) bool {
+						return line.Comments[i].CreatedUnix < line.Comments[j].CreatedUnix
+					})
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// NumFiles returns number of files changes in a diff.
+func (diff *Diff) NumFiles() int {
+	return len(diff.Files)
+}
+
+// Example: @@ -1,8 +1,9 @@ => [..., 1, 8, 1, 9]
+var hunkRegex = regexp.MustCompile(`^@@ -(?P<beginOld>[0-9]+)(,(?P<endOld>[0-9]+))? \+(?P<beginNew>[0-9]+)(,(?P<endNew>[0-9]+))? @@`)
+
+func isHeader(lof string) bool {
+	return strings.HasPrefix(lof, cmdDiffHead) || strings.HasPrefix(lof, "---") || strings.HasPrefix(lof, "+++")
+}
+
+// CutDiffAroundLine cuts a diff of a file in way that only the given line + numberOfLine above it will be shown
+// it also recalculates hunks and adds the appropriate headers to the new diff.
+// Warning: Only one-file diffs are allowed.
+func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLine int) string {
+	if line == 0 || numbersOfLine == 0 {
+		// no line or num of lines => no diff
+		return ""
+	}
+	scanner := bufio.NewScanner(originalDiff)
+	hunk := make([]string, 0)
+	// begin is the start of the hunk containing searched line
+	// end is the end of the hunk ...
+	// currentLine is the line number on the side of the searched line (differentiated by old)
+	// otherLine is the line number on the opposite side of the searched line (differentiated by old)
+	var begin, end, currentLine, otherLine int64
+	var headerLines int
+	for scanner.Scan() {
+		lof := scanner.Text()
+		// Add header to enable parsing
+		if isHeader(lof) {
+			hunk = append(hunk, lof)
+			headerLines++
+		}
+		if currentLine > line {
+			break
+		}
+		// Detect "hunk" with contains commented lof
+		if strings.HasPrefix(lof, "@@") {
+			// Already got our hunk. End of hunk detected!
+			if len(hunk) > headerLines {
+				break
+			}
+			// A map with named groups of our regex to recognize them later more easily
+			submatches := hunkRegex.FindStringSubmatch(lof)
+			groups := make(map[string]string)
+			for i, name := range hunkRegex.SubexpNames() {
+				if i != 0 && name != "" {
+					groups[name] = submatches[i]
+				}
+			}
+			if old {
+				begin = com.StrTo(groups["beginOld"]).MustInt64()
+				end = com.StrTo(groups["endOld"]).MustInt64()
+				// init otherLine with begin of opposite side
+				otherLine = com.StrTo(groups["beginNew"]).MustInt64()
+			} else {
+				begin = com.StrTo(groups["beginNew"]).MustInt64()
+				if groups["endNew"] != "" {
+					end = com.StrTo(groups["endNew"]).MustInt64()
+				} else {
+					end = 0
+				}
+				// init otherLine with begin of opposite side
+				otherLine = com.StrTo(groups["beginOld"]).MustInt64()
+			}
+			end += begin // end is for real only the number of lines in hunk
+			// lof is between begin and end
+			if begin <= line && end >= line {
+				hunk = append(hunk, lof)
+				currentLine = begin
+				continue
+			}
+		} else if len(hunk) > headerLines {
+			hunk = append(hunk, lof)
+			// Count lines in context
+			switch lof[0] {
+			case '+':
+				if !old {
+					currentLine++
+				} else {
+					otherLine++
+				}
+			case '-':
+				if old {
+					currentLine++
+				} else {
+					otherLine++
+				}
+			default:
+				currentLine++
+				otherLine++
+			}
+		}
+	}
+
+	// No hunk found
+	if currentLine == 0 {
+		return ""
+	}
+	// headerLines + hunkLine (1) = totalNonCodeLines
+	if len(hunk)-headerLines-1 <= numbersOfLine {
+		// No need to cut the hunk => return existing hunk
+		return strings.Join(hunk, "\n")
+	}
+	var oldBegin, oldNumOfLines, newBegin, newNumOfLines int64
+	if old {
+		oldBegin = currentLine
+		newBegin = otherLine
+	} else {
+		oldBegin = otherLine
+		newBegin = currentLine
+	}
+	// headers + hunk header
+	newHunk := make([]string, headerLines)
+	// transfer existing headers
+	copy(newHunk, hunk[:headerLines])
+	// transfer last n lines
+	newHunk = append(newHunk, hunk[len(hunk)-numbersOfLine-1:]...)
+	// calculate newBegin, ... by counting lines
+	for i := len(hunk) - 1; i >= len(hunk)-numbersOfLine; i-- {
+		switch hunk[i][0] {
+		case '+':
+			newBegin--
+			newNumOfLines++
+		case '-':
+			oldBegin--
+			oldNumOfLines++
+		default:
+			oldBegin--
+			newBegin--
+			newNumOfLines++
+			oldNumOfLines++
+		}
+	}
+	// construct the new hunk header
+	newHunk[headerLines] = fmt.Sprintf("@@ -%d,%d +%d,%d @@",
+		oldBegin, oldNumOfLines, newBegin, newNumOfLines)
+	return strings.Join(newHunk, "\n")
+}
+
+const cmdDiffHead = "diff --git "
+
+// ParsePatch builds a Diff object from a io.Reader and some
+// parameters.
+// TODO: move this function to gogits/git-module
+func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*Diff, error) {
+	var (
+		diff = &Diff{Files: make([]*DiffFile, 0)}
+
+		curFile    = &DiffFile{}
+		curSection = &DiffSection{
+			Lines: make([]*DiffLine, 0, 10),
+		}
+
+		leftLine, rightLine int
+		lineCount           int
+		curFileLinesCount   int
+		curFileLFSPrefix    bool
+	)
+
+	input := bufio.NewReader(reader)
+	isEOF := false
+	for !isEOF {
+		var linebuf bytes.Buffer
+		for {
+			b, err := input.ReadByte()
+			if err != nil {
+				if err == io.EOF {
+					isEOF = true
+					break
+				} else {
+					return nil, fmt.Errorf("ReadByte: %v", err)
+				}
+			}
+			if b == '\n' {
+				break
+			}
+			if linebuf.Len() < maxLineCharacters {
+				linebuf.WriteByte(b)
+			} else if linebuf.Len() == maxLineCharacters {
+				curFile.IsIncomplete = true
+			}
+		}
+		line := linebuf.String()
+
+		if strings.HasPrefix(line, "+++ ") || strings.HasPrefix(line, "--- ") || len(line) == 0 {
+			continue
+		}
+
+		trimLine := strings.Trim(line, "+- ")
+
+		if trimLine == models.LFSMetaFileIdentifier {
+			curFileLFSPrefix = true
+		}
+
+		if curFileLFSPrefix && strings.HasPrefix(trimLine, models.LFSMetaFileOidPrefix) {
+			oid := strings.TrimPrefix(trimLine, models.LFSMetaFileOidPrefix)
+
+			if len(oid) == 64 {
+				m := &models.LFSMetaObject{Oid: oid}
+				count, err := models.Count(m)
+
+				if err == nil && count > 0 {
+					curFile.IsBin = true
+					curFile.IsLFSFile = true
+					curSection.Lines = nil
+				}
+			}
+		}
+
+		curFileLinesCount++
+		lineCount++
+
+		// Diff data too large, we only show the first about maxLines lines
+		if curFileLinesCount >= maxLines {
+			curFile.IsIncomplete = true
+		}
+		switch {
+		case line[0] == ' ':
+			diffLine := &DiffLine{Type: DiffLinePlain, Content: line, LeftIdx: leftLine, RightIdx: rightLine}
+			leftLine++
+			rightLine++
+			curSection.Lines = append(curSection.Lines, diffLine)
+			continue
+		case line[0] == '@':
+			curSection = &DiffSection{}
+			curFile.Sections = append(curFile.Sections, curSection)
+			ss := strings.Split(line, "@@")
+			diffLine := &DiffLine{Type: DiffLineSection, Content: line}
+			curSection.Lines = append(curSection.Lines, diffLine)
+
+			// Parse line number.
+			ranges := strings.Split(ss[1][1:], " ")
+			leftLine, _ = com.StrTo(strings.Split(ranges[0], ",")[0][1:]).Int()
+			if len(ranges) > 1 {
+				rightLine, _ = com.StrTo(strings.Split(ranges[1], ",")[0]).Int()
+			} else {
+				log.Warn("Parse line number failed: %v", line)
+				rightLine = leftLine
+			}
+			continue
+		case line[0] == '+':
+			curFile.Addition++
+			diff.TotalAddition++
+			diffLine := &DiffLine{Type: DiffLineAdd, Content: line, RightIdx: rightLine}
+			rightLine++
+			curSection.Lines = append(curSection.Lines, diffLine)
+			continue
+		case line[0] == '-':
+			curFile.Deletion++
+			diff.TotalDeletion++
+			diffLine := &DiffLine{Type: DiffLineDel, Content: line, LeftIdx: leftLine}
+			if leftLine > 0 {
+				leftLine++
+			}
+			curSection.Lines = append(curSection.Lines, diffLine)
+		case strings.HasPrefix(line, "Binary"):
+			curFile.IsBin = true
+			continue
+		}
+
+		// Get new file.
+		if strings.HasPrefix(line, cmdDiffHead) {
+			var middle int
+
+			// Note: In case file name is surrounded by double quotes (it happens only in git-shell).
+			// e.g. diff --git "a/xxx" "b/xxx"
+			hasQuote := line[len(cmdDiffHead)] == '"'
+			if hasQuote {
+				middle = strings.Index(line, ` "b/`)
+			} else {
+				middle = strings.Index(line, " b/")
+			}
+
+			beg := len(cmdDiffHead)
+			a := line[beg+2 : middle]
+			b := line[middle+3:]
+
+			if hasQuote {
+				// Keep the entire string in double quotes for now
+				a = line[beg:middle]
+				b = line[middle+1:]
+
+				var err error
+				a, err = strconv.Unquote(a)
+				if err != nil {
+					return nil, fmt.Errorf("Unquote: %v", err)
+				}
+				b, err = strconv.Unquote(b)
+				if err != nil {
+					return nil, fmt.Errorf("Unquote: %v", err)
+				}
+				// Now remove the /a /b
+				a = a[2:]
+				b = b[2:]
+
+			}
+
+			curFile = &DiffFile{
+				Name:      b,
+				OldName:   a,
+				Index:     len(diff.Files) + 1,
+				Type:      DiffFileChange,
+				Sections:  make([]*DiffSection, 0, 10),
+				IsRenamed: a != b,
+			}
+			diff.Files = append(diff.Files, curFile)
+			if len(diff.Files) >= maxFiles {
+				diff.IsIncomplete = true
+				_, err := io.Copy(ioutil.Discard, reader)
+				if err != nil {
+					return nil, fmt.Errorf("Copy: %v", err)
+				}
+				break
+			}
+			curFileLinesCount = 0
+			curFileLFSPrefix = false
+
+			// Check file diff type and is submodule.
+			for {
+				line, err := input.ReadString('\n')
+				if err != nil {
+					if err == io.EOF {
+						isEOF = true
+					} else {
+						return nil, fmt.Errorf("ReadString: %v", err)
+					}
+				}
+
+				switch {
+				case strings.HasPrefix(line, "new file"):
+					curFile.Type = DiffFileAdd
+					curFile.IsCreated = true
+				case strings.HasPrefix(line, "deleted"):
+					curFile.Type = DiffFileDel
+					curFile.IsDeleted = true
+				case strings.HasPrefix(line, "index"):
+					curFile.Type = DiffFileChange
+				case strings.HasPrefix(line, "similarity index 100%"):
+					curFile.Type = DiffFileRename
+				}
+				if curFile.Type > 0 {
+					if strings.HasSuffix(line, " 160000\n") {
+						curFile.IsSubmodule = true
+					}
+					break
+				}
+			}
+		}
+	}
+
+	// FIXME: detect encoding while parsing.
+	var buf bytes.Buffer
+	for _, f := range diff.Files {
+		buf.Reset()
+		for _, sec := range f.Sections {
+			for _, l := range sec.Lines {
+				buf.WriteString(l.Content)
+				buf.WriteString("\n")
+			}
+		}
+		charsetLabel, err := charset.DetectEncoding(buf.Bytes())
+		if charsetLabel != "UTF-8" && err == nil {
+			encoding, _ := stdcharset.Lookup(charsetLabel)
+			if encoding != nil {
+				d := encoding.NewDecoder()
+				for _, sec := range f.Sections {
+					for _, l := range sec.Lines {
+						if c, _, err := transform.String(d, l.Content); err == nil {
+							l.Content = c
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return diff, nil
+}
+
+// GetDiffRange builds a Diff between two commits of a repository.
+// passing the empty string as beforeCommitID returns a diff from the
+// parent commit.
+func GetDiffRange(repoPath, beforeCommitID, afterCommitID string, maxLines, maxLineCharacters, maxFiles int) (*Diff, error) {
+	return GetDiffRangeWithWhitespaceBehavior(repoPath, beforeCommitID, afterCommitID, maxLines, maxLineCharacters, maxFiles, "")
+}
+
+// GetDiffRangeWithWhitespaceBehavior builds a Diff between two commits of a repository.
+// Passing the empty string as beforeCommitID returns a diff from the parent commit.
+// The whitespaceBehavior is either an empty string or a git flag
+func GetDiffRangeWithWhitespaceBehavior(repoPath, beforeCommitID, afterCommitID string, maxLines, maxLineCharacters, maxFiles int, whitespaceBehavior string) (*Diff, error) {
+	gitRepo, err := git.OpenRepository(repoPath)
+	if err != nil {
+		return nil, err
+	}
+
+	commit, err := gitRepo.GetCommit(afterCommitID)
+	if err != nil {
+		return nil, err
+	}
+
+	var cmd *exec.Cmd
+	if len(beforeCommitID) == 0 && commit.ParentCount() == 0 {
+		cmd = exec.Command(git.GitExecutable, "show", afterCommitID)
+	} else {
+		actualBeforeCommitID := beforeCommitID
+		if len(actualBeforeCommitID) == 0 {
+			parentCommit, _ := commit.Parent(0)
+			actualBeforeCommitID = parentCommit.ID.String()
+		}
+		diffArgs := []string{"diff", "-M"}
+		if len(whitespaceBehavior) != 0 {
+			diffArgs = append(diffArgs, whitespaceBehavior)
+		}
+		diffArgs = append(diffArgs, actualBeforeCommitID)
+		diffArgs = append(diffArgs, afterCommitID)
+		cmd = exec.Command(git.GitExecutable, diffArgs...)
+	}
+	cmd.Dir = repoPath
+	cmd.Stderr = os.Stderr
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, fmt.Errorf("StdoutPipe: %v", err)
+	}
+
+	if err = cmd.Start(); err != nil {
+		return nil, fmt.Errorf("Start: %v", err)
+	}
+
+	pid := process.GetManager().Add(fmt.Sprintf("GetDiffRange [repo_path: %s]", repoPath), cmd)
+	defer process.GetManager().Remove(pid)
+
+	diff, err := ParsePatch(maxLines, maxLineCharacters, maxFiles, stdout)
+	if err != nil {
+		return nil, fmt.Errorf("ParsePatch: %v", err)
+	}
+
+	if err = cmd.Wait(); err != nil {
+		return nil, fmt.Errorf("Wait: %v", err)
+	}
+
+	return diff, nil
+}
+
+// RawDiffType type of a raw diff.
+type RawDiffType string
+
+// RawDiffType possible values.
+const (
+	RawDiffNormal RawDiffType = "diff"
+	RawDiffPatch  RawDiffType = "patch"
+)
+
+// GetRawDiff dumps diff results of repository in given commit ID to io.Writer.
+// TODO: move this function to gogits/git-module
+func GetRawDiff(repoPath, commitID string, diffType RawDiffType, writer io.Writer) error {
+	return GetRawDiffForFile(repoPath, "", commitID, diffType, "", writer)
+}
+
+// GetRawDiffForFile dumps diff results of file in given commit ID to io.Writer.
+// TODO: move this function to gogits/git-module
+func GetRawDiffForFile(repoPath, startCommit, endCommit string, diffType RawDiffType, file string, writer io.Writer) error {
+	repo, err := git.OpenRepository(repoPath)
+	if err != nil {
+		return fmt.Errorf("OpenRepository: %v", err)
+	}
+
+	commit, err := repo.GetCommit(endCommit)
+	if err != nil {
+		return fmt.Errorf("GetCommit: %v", err)
+	}
+	fileArgs := make([]string, 0)
+	if len(file) > 0 {
+		fileArgs = append(fileArgs, "--", file)
+	}
+	var cmd *exec.Cmd
+	switch diffType {
+	case RawDiffNormal:
+		if len(startCommit) != 0 {
+			cmd = exec.Command(git.GitExecutable, append([]string{"diff", "-M", startCommit, endCommit}, fileArgs...)...)
+		} else if commit.ParentCount() == 0 {
+			cmd = exec.Command(git.GitExecutable, append([]string{"show", endCommit}, fileArgs...)...)
+		} else {
+			c, _ := commit.Parent(0)
+			cmd = exec.Command(git.GitExecutable, append([]string{"diff", "-M", c.ID.String(), endCommit}, fileArgs...)...)
+		}
+	case RawDiffPatch:
+		if len(startCommit) != 0 {
+			query := fmt.Sprintf("%s...%s", endCommit, startCommit)
+			cmd = exec.Command(git.GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", "--root", query}, fileArgs...)...)
+		} else if commit.ParentCount() == 0 {
+			cmd = exec.Command(git.GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", "--root", endCommit}, fileArgs...)...)
+		} else {
+			c, _ := commit.Parent(0)
+			query := fmt.Sprintf("%s...%s", endCommit, c.ID.String())
+			cmd = exec.Command(git.GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", query}, fileArgs...)...)
+		}
+	default:
+		return fmt.Errorf("invalid diffType: %s", diffType)
+	}
+
+	stderr := new(bytes.Buffer)
+
+	cmd.Dir = repoPath
+	cmd.Stdout = writer
+	cmd.Stderr = stderr
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("Run: %v - %s", err, stderr)
+	}
+	return nil
+}
+
+// GetDiffCommit builds a Diff representing the given commitID.
+func GetDiffCommit(repoPath, commitID string, maxLines, maxLineCharacters, maxFiles int) (*Diff, error) {
+	return GetDiffRange(repoPath, "", commitID, maxLines, maxLineCharacters, maxFiles)
+}
+
+// CommentAsDiff returns c.Patch as *Diff
+func CommentAsDiff(c *models.Comment) (*Diff, error) {
+	diff, err := ParsePatch(setting.Git.MaxGitDiffLines,
+		setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, strings.NewReader(c.Patch))
+	if err != nil {
+		return nil, err
+	}
+	if len(diff.Files) == 0 {
+		return nil, fmt.Errorf("no file found for comment ID: %d", c.ID)
+	}
+	secs := diff.Files[0].Sections
+	if len(secs) == 0 {
+		return nil, fmt.Errorf("no sections found for comment ID: %d", c.ID)
+	}
+	return diff, nil
+}
+
+// CommentMustAsDiff executes AsDiff and logs the error instead of returning
+func CommentMustAsDiff(c *models.Comment) *Diff {
+	diff, err := CommentAsDiff(c)
+	if err != nil {
+		log.Warn("CommentMustAsDiff: %v", err)
+	}
+	return diff
+}
diff --git a/services/gitdiff/gitdiff_test.go b/services/gitdiff/gitdiff_test.go
new file mode 100644
index 0000000000..acf3c6d7bd
--- /dev/null
+++ b/services/gitdiff/gitdiff_test.go
@@ -0,0 +1,197 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package gitdiff
+
+import (
+	"html/template"
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/setting"
+
+	dmp "github.com/sergi/go-diff/diffmatchpatch"
+	"github.com/stretchr/testify/assert"
+)
+
+func assertEqual(t *testing.T, s1 string, s2 template.HTML) {
+	if s1 != string(s2) {
+		t.Errorf("%s should be equal %s", s2, s1)
+	}
+}
+
+func TestDiffToHTML(t *testing.T) {
+	assertEqual(t, "foo <span class=\"added-code\">bar</span> biz", diffToHTML([]dmp.Diff{
+		{Type: dmp.DiffEqual, Text: "foo "},
+		{Type: dmp.DiffInsert, Text: "bar"},
+		{Type: dmp.DiffDelete, Text: " baz"},
+		{Type: dmp.DiffEqual, Text: " biz"},
+	}, DiffLineAdd))
+
+	assertEqual(t, "foo <span class=\"removed-code\">bar</span> biz", diffToHTML([]dmp.Diff{
+		{Type: dmp.DiffEqual, Text: "foo "},
+		{Type: dmp.DiffDelete, Text: "bar"},
+		{Type: dmp.DiffInsert, Text: " baz"},
+		{Type: dmp.DiffEqual, Text: " biz"},
+	}, DiffLineDel))
+}
+
+const exampleDiff = `diff --git a/README.md b/README.md
+--- a/README.md
++++ b/README.md
+@@ -1,3 +1,6 @@
+ # gitea-github-migrator
++
++ Build Status
+- Latest Release
+ Docker Pulls
++ cut off
++ cut off`
+
+func TestCutDiffAroundLine(t *testing.T) {
+	result := CutDiffAroundLine(strings.NewReader(exampleDiff), 4, false, 3)
+	resultByLine := strings.Split(result, "\n")
+	assert.Len(t, resultByLine, 7)
+	// Check if headers got transferred
+	assert.Equal(t, "diff --git a/README.md b/README.md", resultByLine[0])
+	assert.Equal(t, "--- a/README.md", resultByLine[1])
+	assert.Equal(t, "+++ b/README.md", resultByLine[2])
+	// Check if hunk header is calculated correctly
+	assert.Equal(t, "@@ -2,2 +3,2 @@", resultByLine[3])
+	// Check if line got transferred
+	assert.Equal(t, "+ Build Status", resultByLine[4])
+
+	// Must be same result as before since old line 3 == new line 5
+	newResult := CutDiffAroundLine(strings.NewReader(exampleDiff), 3, true, 3)
+	assert.Equal(t, result, newResult, "Must be same result as before since old line 3 == new line 5")
+
+	newResult = CutDiffAroundLine(strings.NewReader(exampleDiff), 6, false, 300)
+	assert.Equal(t, exampleDiff, newResult)
+
+	emptyResult := CutDiffAroundLine(strings.NewReader(exampleDiff), 6, false, 0)
+	assert.Empty(t, emptyResult)
+
+	// Line is out of scope
+	emptyResult = CutDiffAroundLine(strings.NewReader(exampleDiff), 434, false, 0)
+	assert.Empty(t, emptyResult)
+}
+
+func BenchmarkCutDiffAroundLine(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		CutDiffAroundLine(strings.NewReader(exampleDiff), 3, true, 3)
+	}
+}
+
+func ExampleCutDiffAroundLine() {
+	const diff = `diff --git a/README.md b/README.md
+--- a/README.md
++++ b/README.md
+@@ -1,3 +1,6 @@
+ # gitea-github-migrator
++
++ Build Status
+- Latest Release
+ Docker Pulls
++ cut off
++ cut off`
+	result := CutDiffAroundLine(strings.NewReader(diff), 4, false, 3)
+	println(result)
+}
+
+func TestParsePatch(t *testing.T) {
+	var diff = `diff --git "a/README.md" "b/README.md"
+--- a/README.md
++++ b/README.md
+@@ -1,3 +1,6 @@
+ # gitea-github-migrator
++
++ Build Status
+- Latest Release
+ Docker Pulls
++ cut off
++ cut off`
+	result, err := ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, strings.NewReader(diff))
+	if err != nil {
+		t.Errorf("ParsePatch failed: %s", err)
+	}
+	println(result)
+
+	var diff2 = `diff --git "a/A \\ B" "b/A \\ B"
+--- "a/A \\ B"
++++ "b/A \\ B"
+@@ -1,3 +1,6 @@
+ # gitea-github-migrator
++
++ Build Status
+- Latest Release
+ Docker Pulls
++ cut off
++ cut off`
+	result, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, strings.NewReader(diff2))
+	if err != nil {
+		t.Errorf("ParsePatch failed: %s", err)
+	}
+	println(result)
+
+	var diff3 = `diff --git a/README.md b/README.md
+--- a/README.md
++++ b/README.md
+@@ -1,3 +1,6 @@
+ # gitea-github-migrator
++
++ Build Status
+- Latest Release
+ Docker Pulls
++ cut off
++ cut off`
+	result, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, strings.NewReader(diff3))
+	if err != nil {
+		t.Errorf("ParsePatch failed: %s", err)
+	}
+	println(result)
+}
+
+func setupDefaultDiff() *Diff {
+	return &Diff{
+		Files: []*DiffFile{
+			{
+				Name: "README.md",
+				Sections: []*DiffSection{
+					{
+						Lines: []*DiffLine{
+							{
+								LeftIdx:  4,
+								RightIdx: 4,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+func TestDiff_LoadComments(t *testing.T) {
+	assert.NoError(t, models.PrepareTestDatabase())
+
+	issue := models.AssertExistsAndLoadBean(t, &models.Issue{ID: 2}).(*models.Issue)
+	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
+	diff := setupDefaultDiff()
+	assert.NoError(t, diff.LoadComments(issue, user))
+	assert.Len(t, diff.Files[0].Sections[0].Lines[0].Comments, 2)
+}
+
+func TestDiffLine_CanComment(t *testing.T) {
+	assert.False(t, (&DiffLine{Type: DiffLineSection}).CanComment())
+	assert.False(t, (&DiffLine{Type: DiffLineAdd, Comments: []*models.Comment{{Content: "bla"}}}).CanComment())
+	assert.True(t, (&DiffLine{Type: DiffLineAdd}).CanComment())
+	assert.True(t, (&DiffLine{Type: DiffLineDel}).CanComment())
+	assert.True(t, (&DiffLine{Type: DiffLinePlain}).CanComment())
+}
+
+func TestDiffLine_GetCommentSide(t *testing.T) {
+	assert.Equal(t, "previous", (&DiffLine{Comments: []*models.Comment{{Line: -3}}}).GetCommentSide())
+	assert.Equal(t, "proposed", (&DiffLine{Comments: []*models.Comment{{Line: 3}}}).GetCommentSide())
+}
diff --git a/services/gitdiff/main_test.go b/services/gitdiff/main_test.go
new file mode 100644
index 0000000000..5ed3c75b70
--- /dev/null
+++ b/services/gitdiff/main_test.go
@@ -0,0 +1,16 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package gitdiff
+
+import (
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+)
+
+func TestMain(m *testing.M) {
+	models.MainTest(m, filepath.Join("..", ".."))
+}