Ensure syntax highlighting is the same inside diffs (#12205)

Make sure to end up with the same syntax highlighting inside various sections of diffs by processing the code first before detecting specific changes between the lines. Also try and make sure that when highlighting individual lines in a diff that it is tokenized the same as it would be when part of an entire file with more context.

Fixes: #12190

2 files changed, 12 insertions, 8 deletions

diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go
index 31448ccd63..a2bf93ee92 100644
--- a/modules/highlight/highlight.go
+++ b/modules/highlight/highlight.go
@@ -14,7 +14,6 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"github.com/alecthomas/chroma"
 	"github.com/alecthomas/chroma/formatters/html"
 	"github.com/alecthomas/chroma/lexers"
 	"github.com/alecthomas/chroma/styles"
@@ -44,9 +43,12 @@ func NewContext() {
 func Code(fileName, code string) string {
 	NewContext()
 
-	if code == "" {
+	// diff view newline will be passed as empty, change to literal \n so it can be copied
+	// preserve literal newline in blame view
+	if code == "" || code == "\n" {
 		return "\n"
 	}
+
 	if len(code) > sizeLimit {
 		return code
 	}
@@ -72,7 +74,7 @@ func Code(fileName, code string) string {
 		lexer = lexers.Fallback
 	}
 
-	iterator, err := lexer.Tokenise(&chroma.TokeniseOptions{State: "root", Nested: true}, string(code))
+	iterator, err := lexer.Tokenise(nil, string(code))
 	if err != nil {
 		log.Error("Can't tokenize code: %v", err)
 		return code
@@ -85,7 +87,9 @@ func Code(fileName, code string) string {
 	}
 
 	htmlw.Flush()
-	return htmlbuf.String()
+	// Chroma will add newlines for certain lexers in order to highlight them properly
+	// Once highlighted, strip them here so they don't cause copy/paste trouble in HTML output
+	return strings.TrimSuffix(htmlbuf.String(), "\n")
 }
 
 // File returns map with line lumbers and HTML version of code with chroma syntax highlighting classes
diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go
index d335661ffb..7d08f63eb7 100644
--- a/services/gitdiff/gitdiff.go
+++ b/services/gitdiff/gitdiff.go
@@ -188,14 +188,14 @@ func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineT
 		switch {
 		case diffs[i].Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd:
 			buf.Write(addedCodePrefix)
-			buf.WriteString(highlight.Code(fileName, diffs[i].Text))
+			buf.WriteString(getLineContent(diffs[i].Text))
 			buf.Write(codeTagSuffix)
 		case diffs[i].Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel:
 			buf.Write(removedCodePrefix)
-			buf.WriteString(highlight.Code(fileName, diffs[i].Text))
+			buf.WriteString(getLineContent(diffs[i].Text))
 			buf.Write(codeTagSuffix)
 		case diffs[i].Type == diffmatchpatch.DiffEqual:
-			buf.WriteString(highlight.Code(fileName, getLineContent(diffs[i].Text)))
+			buf.WriteString(getLineContent(diffs[i].Text))
 		}
 	}
 	return template.HTML(buf.Bytes())
@@ -287,7 +287,7 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) tem
 		return template.HTML(highlight.Code(diffSection.FileName, diffLine.Content))
 	}
 
-	diffRecord := diffMatchPatch.DiffMain(diff1[1:], diff2[1:], true)
+	diffRecord := diffMatchPatch.DiffMain(highlight.Code(diffSection.FileName, diff1[1:]), highlight.Code(diffSection.FileName, diff2[1:]), true)
 	diffRecord = diffMatchPatch.DiffCleanupEfficiency(diffRecord)
 	return diffToHTML(diffSection.FileName, diffRecord, diffLine.Type)
 }