Server-side syntax highlighting for all code (#12047)

* Server-side syntax hilighting for all code

This PR does a few things:

* Remove all traces of highlight.js
* Use chroma library to provide fast syntax hilighting directly on the server
* Provide syntax hilighting for diffs
* Re-style both unified and split diffs views
* Add custom syntax hilighting styling for both regular and arc-green

Fixes #7729
Fixes #10157
Fixes #11825
Fixes #7728
Fixes #3872
Fixes #3682

And perhaps gets closer to #9553

* fix line marker

* fix repo search

* Fix single line select

* properly load settings

* npm uninstall highlight.js

* review suggestion

* code review

* forgot to call function

* fix test

* Apply suggestions from code review

suggestions from @silverwind thanks

Co-authored-by: silverwind <me@silverwind.io>

* code review

* copy/paste error

* Use const for highlight size limit

* Update web_src/less/_repository.less

Co-authored-by: Lauris BH <lauris@nix.lv>

* update size limit to 1MB and other styling tweaks

* fix highlighting for certain diff sections

* fix test

* add worker back as suggested

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Lauris BH <lauris@nix.lv>

2 files changed, 28 insertions, 26 deletions

diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go
index 02aef70882..80e6eb1ecd 100644
--- a/services/gitdiff/gitdiff.go
+++ b/services/gitdiff/gitdiff.go
@@ -10,7 +10,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"html"
 	"html/template"
 	"io"
 	"io/ioutil"
@@ -164,15 +163,16 @@ func getDiffLineSectionInfo(treePath, line string, lastLeftIdx, lastRightIdx int
 // escape a line's content or return <br> needed for copy/paste purposes
 func getLineContent(content string) string {
 	if len(content) > 0 {
-		return html.EscapeString(content)
+		return content
 	}
-	return "<br>"
+	return "\n"
 }
 
 // DiffSection represents a section of a DiffFile.
 type DiffSection struct {
-	Name  string
-	Lines []*DiffLine
+	FileName string
+	Name     string
+	Lines    []*DiffLine
 }
 
 var (
@@ -181,24 +181,23 @@ var (
 	codeTagSuffix     = []byte(`</span>`)
 )
 
-func diffToHTML(diffs []diffmatchpatch.Diff, lineType DiffLineType) template.HTML {
+func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineType) template.HTML {
 	buf := bytes.NewBuffer(nil)
 
 	for i := range diffs {
 		switch {
 		case diffs[i].Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd:
 			buf.Write(addedCodePrefix)
-			buf.WriteString(getLineContent(diffs[i].Text))
+			buf.WriteString(highlight.Code(fileName, diffs[i].Text))
 			buf.Write(codeTagSuffix)
 		case diffs[i].Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel:
 			buf.Write(removedCodePrefix)
-			buf.WriteString(getLineContent(diffs[i].Text))
+			buf.WriteString(highlight.Code(fileName, diffs[i].Text))
 			buf.Write(codeTagSuffix)
 		case diffs[i].Type == diffmatchpatch.DiffEqual:
-			buf.WriteString(getLineContent(diffs[i].Text))
+			buf.WriteString(highlight.Code(fileName, getLineContent(diffs[i].Text)))
 		}
 	}
-
 	return template.HTML(buf.Bytes())
 }
 
@@ -256,6 +255,7 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) tem
 	if setting.Git.DisableDiffHighlight {
 		return template.HTML(getLineContent(diffLine.Content[1:]))
 	}
+
 	var (
 		compareDiffLine *DiffLine
 		diff1           string
@@ -264,31 +264,32 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) tem
 
 	// try to find equivalent diff line. ignore, otherwise
 	switch diffLine.Type {
+	case DiffLineSection:
+		return template.HTML(getLineContent(diffLine.Content[1:]))
 	case DiffLineAdd:
 		compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx)
 		if compareDiffLine == nil {
-			return template.HTML(getLineContent(diffLine.Content[1:]))
+			return template.HTML(highlight.Code(diffSection.FileName, diffLine.Content[1:]+"\n"))
 		}
 		diff1 = compareDiffLine.Content
 		diff2 = diffLine.Content
 	case DiffLineDel:
 		compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx)
 		if compareDiffLine == nil {
-			return template.HTML(getLineContent(diffLine.Content[1:]))
+			return template.HTML(highlight.Code(diffSection.FileName, diffLine.Content[1:]+"\n"))
 		}
 		diff1 = diffLine.Content
 		diff2 = compareDiffLine.Content
 	default:
 		if strings.IndexByte(" +-", diffLine.Content[0]) > -1 {
-			return template.HTML(getLineContent(diffLine.Content[1:]))
+			return template.HTML(highlight.Code(diffSection.FileName, diffLine.Content[1:]+"\n"))
 		}
-		return template.HTML(getLineContent(diffLine.Content))
+		return template.HTML(highlight.Code(diffSection.FileName, diffLine.Content))
 	}
 
 	diffRecord := diffMatchPatch.DiffMain(diff1[1:], diff2[1:], true)
 	diffRecord = diffMatchPatch.DiffCleanupEfficiency(diffRecord)
-
-	return diffToHTML(diffRecord, diffLine.Type)
+	return diffToHTML(diffSection.FileName, diffRecord, diffLine.Type)
 }
 
 // DiffFile represents a file diff.
@@ -313,11 +314,6 @@ func (diffFile *DiffFile) GetType() int {
 	return int(diffFile.Type)
 }
 
-// GetHighlightClass returns highlight class for a filename.
-func (diffFile *DiffFile) GetHighlightClass() string {
-	return highlight.FileNameToHighlightClass(diffFile.Name)
-}
-
 // GetTailSection creates a fake DiffLineSection if the last section is not the end of the file
 func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommitID, rightCommitID string) *DiffSection {
 	if len(diffFile.Sections) == 0 || diffFile.Type != DiffFileChange || diffFile.IsBin || diffFile.IsLFSFile {
@@ -348,7 +344,7 @@ func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommitID,
 			LeftIdx:      leftLineCount,
 			RightIdx:     rightLineCount,
 		}}
-	tailSection := &DiffSection{Lines: []*DiffLine{tailDiffLine}}
+	tailSection := &DiffSection{FileName: diffFile.Name, Lines: []*DiffLine{tailDiffLine}}
 	return tailSection
 
 }
@@ -405,8 +401,7 @@ const cmdDiffHead = "diff --git "
 // TODO: move this function to gogits/git-module
 func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*Diff, error) {
 	var (
-		diff = &Diff{Files: make([]*DiffFile, 0)}
-
+		diff       = &Diff{Files: make([]*DiffFile, 0)}
 		curFile    = &DiffFile{}
 		curSection = &DiffSection{
 			Lines: make([]*DiffLine, 0, 10),
@@ -481,6 +476,7 @@ func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*D
 			leftLine++
 			rightLine++
 			curSection.Lines = append(curSection.Lines, diffLine)
+			curSection.FileName = curFile.Name
 			continue
 		case line[0] == '@':
 			curSection = &DiffSection{}
@@ -492,6 +488,7 @@ func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*D
 				SectionInfo: lineSectionInfo,
 			}
 			curSection.Lines = append(curSection.Lines, diffLine)
+			curSection.FileName = curFile.Name
 			// update line number.
 			leftLine = lineSectionInfo.LeftIdx
 			rightLine = lineSectionInfo.RightIdx
@@ -502,6 +499,7 @@ func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*D
 			diffLine := &DiffLine{Type: DiffLineAdd, Content: line, RightIdx: rightLine}
 			rightLine++
 			curSection.Lines = append(curSection.Lines, diffLine)
+			curSection.FileName = curFile.Name
 			continue
 		case line[0] == '-':
 			curFile.Deletion++
@@ -511,6 +509,7 @@ func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*D
 				leftLine++
 			}
 			curSection.Lines = append(curSection.Lines, diffLine)
+			curSection.FileName = curFile.Name
 		case strings.HasPrefix(line, "Binary"):
 			curFile.IsBin = true
 			continue
diff --git a/services/gitdiff/gitdiff_test.go b/services/gitdiff/gitdiff_test.go
index efdf439933..0dc3f06155 100644
--- a/services/gitdiff/gitdiff_test.go
+++ b/services/gitdiff/gitdiff_test.go
@@ -15,6 +15,8 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/setting"
 
+	"gopkg.in/ini.v1"
+
 	dmp "github.com/sergi/go-diff/diffmatchpatch"
 	"github.com/stretchr/testify/assert"
 )
@@ -26,14 +28,15 @@ func assertEqual(t *testing.T, s1 string, s2 template.HTML) {
 }
 
 func TestDiffToHTML(t *testing.T) {
-	assertEqual(t, "foo <span class=\"added-code\">bar</span> biz", diffToHTML([]dmp.Diff{
+	setting.Cfg = ini.Empty()
+	assertEqual(t, "foo <span class=\"added-code\">bar</span> biz", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "foo "},
 		{Type: dmp.DiffInsert, Text: "bar"},
 		{Type: dmp.DiffDelete, Text: " baz"},
 		{Type: dmp.DiffEqual, Text: " biz"},
 	}, DiffLineAdd))
 
-	assertEqual(t, "foo <span class=\"removed-code\">bar</span> biz", diffToHTML([]dmp.Diff{
+	assertEqual(t, "foo <span class=\"removed-code\">bar</span> biz", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "foo "},
 		{Type: dmp.DiffDelete, Text: "bar"},
 		{Type: dmp.DiffInsert, Text: " baz"},