Server-side syntax highlighting for all code (#12047)

* Server-side syntax hilighting for all code

This PR does a few things:

* Remove all traces of highlight.js
* Use chroma library to provide fast syntax hilighting directly on the server
* Provide syntax hilighting for diffs
* Re-style both unified and split diffs views
* Add custom syntax hilighting styling for both regular and arc-green

Fixes #7729
Fixes #10157
Fixes #11825
Fixes #7728
Fixes #3872
Fixes #3682

And perhaps gets closer to #9553

* fix line marker

* fix repo search

* Fix single line select

* properly load settings

* npm uninstall highlight.js

* review suggestion

* code review

* forgot to call function

* fix test

* Apply suggestions from code review

suggestions from @silverwind thanks

Co-authored-by: silverwind <me@silverwind.io>

* code review

* copy/paste error

* Use const for highlight size limit

* Update web_src/less/_repository.less

Co-authored-by: Lauris BH <lauris@nix.lv>

* update size limit to 1MB and other styling tweaks

* fix highlighting for certain diff sections

* fix test

* add worker back as suggested

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Lauris BH <lauris@nix.lv>

1 files changed, 116 insertions, 119 deletions

diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go
index ffd88656ae..90590d220b 100644
--- a/modules/highlight/highlight.go
+++ b/modules/highlight/highlight.go
@@ -1,151 +1,148 @@
 // Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2020 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
 package highlight
 
 import (
-	"path"
+	"bufio"
+	"bytes"
+	"path/filepath"
 	"strings"
+	"sync"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"github.com/alecthomas/chroma/formatters/html"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
 )
 
+// don't index files larger than this many bytes for performance purposes
+const sizeLimit = 1000000
+
 var (
-	// File name should ignore highlight.
-	ignoreFileNames = map[string]bool{
-		"license": true,
-		"copying": true,
-	}
+	// For custom user mapping
+	highlightMapping = map[string]string{}
+
+	once sync.Once
+)
+
+// NewContext loads custom highlight map from local config
+func NewContext() {
+	once.Do(func() {
+		keys := setting.Cfg.Section("highlight.mapping").Keys()
+		for i := range keys {
+			highlightMapping[keys[i].Name()] = keys[i].Value()
+		}
+	})
+}
+
+// Code returns a HTML version of code string with chroma syntax highlighting classes
+func Code(fileName, code string) string {
+	NewContext()
 
-	// File names that are representing highlight classes.
-	highlightFileNames = map[string]string{
-		"dockerfile":     "dockerfile",
-		"makefile":       "makefile",
-		"gnumakefile":    "makefile",
-		"cmakelists.txt": "cmake",
+	if len(code) > sizeLimit {
+		return code
+	}
+	formatter := html.New(html.WithClasses(true),
+		html.WithLineNumbers(false),
+		html.PreventSurroundingPre(true),
+	)
+	if formatter == nil {
+		log.Error("Couldn't create chroma formatter")
+		return code
 	}
 
-	// Extensions that are same as highlight classes.
-	// See hljs.listLanguages() for list of language names.
-	highlightExts = map[string]struct{}{
-		".applescript": {},
-		".arm":         {},
-		".as":          {},
-		".bash":        {},
-		".bat":         {},
-		".c":           {},
-		".cmake":       {},
-		".cpp":         {},
-		".cs":          {},
-		".css":         {},
-		".dart":        {},
-		".diff":        {},
-		".django":      {},
-		".go":          {},
-		".gradle":      {},
-		".groovy":      {},
-		".haml":        {},
-		".handlebars":  {},
-		".html":        {},
-		".ini":         {},
-		".java":        {},
-		".json":        {},
-		".less":        {},
-		".lua":         {},
-		".php":         {},
-		".scala":       {},
-		".scss":        {},
-		".sql":         {},
-		".swift":       {},
-		".ts":          {},
-		".xml":         {},
-		".yaml":        {},
+	htmlbuf := bytes.Buffer{}
+	htmlw := bufio.NewWriter(&htmlbuf)
+
+	if val, ok := highlightMapping[filepath.Ext(fileName)]; ok {
+		//change file name to one with mapped extension so we look that up instead
+		fileName = "mapped." + val
 	}
 
-	// Extensions that are not same as highlight classes.
-	highlightMapping = map[string]string{
-		".ahk":     "autohotkey",
-		".crmsh":   "crmsh",
-		".dash":    "shell",
-		".erl":     "erlang",
-		".escript": "erlang",
-		".ex":      "elixir",
-		".exs":     "elixir",
-		".f":       "fortran",
-		".f77":     "fortran",
-		".f90":     "fortran",
-		".f95":     "fortran",
-		".feature": "gherkin",
-		".fish":    "shell",
-		".for":     "fortran",
-		".hbs":     "handlebars",
-		".hs":      "haskell",
-		".hx":      "haxe",
-		".js":      "javascript",
-		".jsx":     "javascript",
-		".ksh":     "shell",
-		".kt":      "kotlin",
-		".l":       "ocaml",
-		".ls":      "livescript",
-		".md":      "markdown",
-		".mjs":     "javascript",
-		".mli":     "ocaml",
-		".mll":     "ocaml",
-		".mly":     "ocaml",
-		".patch":   "diff",
-		".pl":      "perl",
-		".pm":      "perl",
-		".ps1":     "powershell",
-		".psd1":    "powershell",
-		".psm1":    "powershell",
-		".py":      "python",
-		".pyw":     "python",
-		".rb":      "ruby",
-		".rs":      "rust",
-		".scpt":    "applescript",
-		".scptd":   "applescript",
-		".sh":      "bash",
-		".tcsh":    "shell",
-		".ts":      "typescript",
-		".tsx":     "typescript",
-		".txt":     "plaintext",
-		".vb":      "vbnet",
-		".vbs":     "vbscript",
-		".yml":     "yaml",
-		".zsh":     "shell",
+	lexer := lexers.Match(fileName)
+	if lexer == nil {
+		lexer = lexers.Fallback
 	}
-)
 
-// NewContext loads highlight map
-func NewContext() {
-	keys := setting.Cfg.Section("highlight.mapping").Keys()
-	for i := range keys {
-		highlightMapping[keys[i].Name()] = keys[i].Value()
+	iterator, err := lexer.Tokenise(nil, string(code))
+	if err != nil {
+		log.Error("Can't tokenize code: %v", err)
+		return code
 	}
+	// style not used for live site but need to pass something
+	err = formatter.Format(htmlw, styles.GitHub, iterator)
+	if err != nil {
+		log.Error("Can't format code: %v", err)
+		return code
+	}
+
+	htmlw.Flush()
+	return htmlbuf.String()
 }
 
-// FileNameToHighlightClass returns the best match for highlight class name
-// based on the rule of highlight.js.
-func FileNameToHighlightClass(fname string) string {
-	fname = strings.ToLower(fname)
-	if ignoreFileNames[fname] {
-		return "nohighlight"
+// File returns map with line lumbers and HTML version of code with chroma syntax highlighting classes
+func File(numLines int, fileName string, code []byte) map[int]string {
+	NewContext()
+
+	if len(code) > sizeLimit {
+		return plainText(string(code), numLines)
+	}
+	formatter := html.New(html.WithClasses(true),
+		html.WithLineNumbers(false),
+		html.PreventSurroundingPre(true),
+	)
+
+	if formatter == nil {
+		log.Error("Couldn't create chroma formatter")
+		return plainText(string(code), numLines)
+	}
+
+	htmlbuf := bytes.Buffer{}
+	htmlw := bufio.NewWriter(&htmlbuf)
+
+	if val, ok := highlightMapping[filepath.Ext(fileName)]; ok {
+		fileName = "test." + val
+	}
+
+	lexer := lexers.Match(fileName)
+	if lexer == nil {
+		lexer = lexers.Analyse(string(code))
+		if lexer == nil {
+			lexer = lexers.Fallback
+		}
 	}
 
-	if name, ok := highlightFileNames[fname]; ok {
-		return name
+	iterator, err := lexer.Tokenise(nil, string(code))
+	if err != nil {
+		log.Error("Can't tokenize code: %v", err)
+		return plainText(string(code), numLines)
 	}
 
-	ext := path.Ext(fname)
-	if _, ok := highlightExts[ext]; ok {
-		return ext[1:]
+	err = formatter.Format(htmlw, styles.GitHub, iterator)
+	if err != nil {
+		log.Error("Can't format code: %v", err)
+		return plainText(string(code), numLines)
 	}
 
-	name, ok := highlightMapping[ext]
-	if ok {
-		return name
+	htmlw.Flush()
+	m := make(map[int]string, numLines)
+	for k, v := range strings.SplitN(htmlbuf.String(), "\n", numLines) {
+		line := k + 1
+		m[line] = string(v)
 	}
+	return m
+}
 
-	return ""
+// return unhiglighted map
+func plainText(code string, numLines int) map[int]string {
+	m := make(map[int]string, numLines)
+	for k, v := range strings.SplitN(string(code), "\n", numLines) {
+		line := k + 1
+		m[line] = string(v)
+	}
+	return m
 }