Add option to disable ambiguous unicode characters detection (#28454) (#28499)

Backport #28454 (the only conflict is caused by some comments)

* Close #24483
* Close #28123
* Close #23682
* Close #23149

2 files changed, 12 insertions, 13 deletions

diff --git a/routers/web/repo/blame.go b/routers/web/repo/blame.go
index 52d350ff66..f07086ef1c 100644
--- a/routers/web/repo/blame.go
+++ b/routers/web/repo/blame.go
@@ -310,8 +310,7 @@ func renderBlame(ctx *context.Context, blameParts []git.BlamePart, commitNames m
 				lexerName = lexerNameForLine
 			}
 
-			br.EscapeStatus, line = charset.EscapeControlHTML(line, ctx.Locale)
-			br.Code = gotemplate.HTML(line)
+			br.EscapeStatus, br.Code = charset.EscapeControlHTML(line, ctx.Locale)
 			rows = append(rows, br)
 			escapeStatus = escapeStatus.Or(br.EscapeStatus)
 		}
diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go
index 9f2c316fbf..b17d11ec63 100644
--- a/routers/web/repo/view.go
+++ b/routers/web/repo/view.go
@@ -9,6 +9,7 @@ import (
 	gocontext "context"
 	"encoding/base64"
 	"fmt"
+	"html/template"
 	"image"
 	"io"
 	"net/http"
@@ -317,19 +318,18 @@ func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.Tr
 		}, rd)
 		if err != nil {
 			log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.Name(), ctx.Repo.Repository, err)
-			buf := &bytes.Buffer{}
-			ctx.Data["EscapeStatus"], _ = charset.EscapeControlStringReader(rd, buf, ctx.Locale)
-			ctx.Data["FileContent"] = buf.String()
+			delete(ctx.Data, "IsMarkup")
 		}
-	} else {
+	}
+
+	if ctx.Data["IsMarkup"] != true {
 		ctx.Data["IsPlainText"] = true
-		buf := &bytes.Buffer{}
-		ctx.Data["EscapeStatus"], err = charset.EscapeControlStringReader(rd, buf, ctx.Locale)
+		content, err := io.ReadAll(rd)
 		if err != nil {
-			log.Error("Read failed: %v", err)
+			log.Error("Read readme content failed: %v", err)
 		}
-
-		ctx.Data["FileContent"] = buf.String()
+		contentEscaped := template.HTMLEscapeString(util.UnsafeBytesToString(content))
+		ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlHTML(template.HTML(contentEscaped), ctx.Locale)
 	}
 }
 
@@ -611,7 +611,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 	}
 }
 
-func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input io.Reader) (escaped *charset.EscapeStatus, output string, err error) {
+func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input io.Reader) (escaped *charset.EscapeStatus, output template.HTML, err error) {
 	markupRd, markupWr := io.Pipe()
 	defer markupWr.Close()
 	done := make(chan struct{})
@@ -619,7 +619,7 @@ func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input i
 		sb := &strings.Builder{}
 		// We allow NBSP here this is rendered
 		escaped, _ = charset.EscapeControlReader(markupRd, sb, ctx.Locale, charset.RuneNBSP)
-		output = sb.String()
+		output = template.HTML(sb.String())
 		close(done)
 	}()
 	err = markup.Render(renderCtx, input, markupWr)