Preserve BOM in web editor (#28935)

The `ToUTF8*` functions were stripping BOM, while BOM is actually valid
in UTF8, so the stripping must be optional depending on use case. This
does:

- Add a options struct to all `ToUTF8*` functions, that by default will
strip BOM to preserve existing behaviour
- Remove `ToUTF8` function, it was dead code
- Rename `ToUTF8WithErr` to `ToUTF8`
- Preserve BOM in Monaco Editor
- Remove a unnecessary newline in the textarea value. Browsers did
ignore it, it seems but it's better not to rely on this behaviour.

Fixes: https://github.com/go-gitea/gitea/issues/28743
Related: https://github.com/go-gitea/gitea/issues/6716 which seems to
have once introduced a mechanism that strips and re-adds the BOM, but
from what I can tell, this mechanism was removed at some point after
that PR.

6 files changed, 8 insertions, 8 deletions

diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go
index abb39caa57..00157d44c9 100644
--- a/routers/web/repo/commit.go
+++ b/routers/web/repo/commit.go
@@ -384,7 +384,7 @@ func Diff(ctx *context.Context) {
 			Metas:   ctx.Repo.Repository.ComposeMetas(ctx),
 			GitRepo: ctx.Repo.GitRepo,
 			Ctx:     ctx,
-		}, template.HTMLEscapeString(string(charset.ToUTF8WithFallback(note.Message))))
+		}, template.HTMLEscapeString(string(charset.ToUTF8WithFallback(note.Message, charset.ConvertOpts{}))))
 		if err != nil {
 			ctx.ServerError("RenderCommitMessage", err)
 			return
diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go
index 042b8ed692..5ae48e1ce8 100644
--- a/routers/web/repo/compare.go
+++ b/routers/web/repo/compare.go
@@ -142,7 +142,7 @@ func setCsvCompareContext(ctx *context.Context) {
 				return nil, nil, err
 			}
 
-			csvReader, err := csv_module.CreateReaderAndDetermineDelimiter(ctx, charset.ToUTF8WithFallbackReader(reader))
+			csvReader, err := csv_module.CreateReaderAndDetermineDelimiter(ctx, charset.ToUTF8WithFallbackReader(reader, charset.ConvertOpts{}))
 			return csvReader, reader, err
 		}
 
diff --git a/routers/web/repo/editor.go b/routers/web/repo/editor.go
index 5e7cd1caa3..85d40e7820 100644
--- a/routers/web/repo/editor.go
+++ b/routers/web/repo/editor.go
@@ -166,8 +166,8 @@ func editFile(ctx *context.Context, isNewFile bool) {
 		}
 
 		buf = append(buf, d...)
-		if content, err := charset.ToUTF8WithErr(buf); err != nil {
-			log.Error("ToUTF8WithErr: %v", err)
+		if content, err := charset.ToUTF8(buf, charset.ConvertOpts{KeepBOM: true}); err != nil {
+			log.Error("ToUTF8: %v", err)
 			ctx.Data["FileContent"] = string(buf)
 		} else {
 			ctx.Data["FileContent"] = content
diff --git a/routers/web/repo/render.go b/routers/web/repo/render.go
index f2c6ab3f8f..7eb5a42aa4 100644
--- a/routers/web/repo/render.go
+++ b/routers/web/repo/render.go
@@ -43,7 +43,7 @@ func RenderFile(ctx *context.Context) {
 	st := typesniffer.DetectContentType(buf)
 	isTextFile := st.IsText()
 
-	rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
+	rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc), charset.ConvertOpts{})
 
 	if markupType := markup.Type(blob.Name()); markupType == "" {
 		if isTextFile {
diff --git a/routers/web/repo/setting/lfs.go b/routers/web/repo/setting/lfs.go
index edf1298c20..cd0f11d548 100644
--- a/routers/web/repo/setting/lfs.go
+++ b/routers/web/repo/setting/lfs.go
@@ -303,7 +303,7 @@ func LFSFileGet(ctx *context.Context) {
 			break
 		}
 
-		rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
+		rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc), charset.ConvertOpts{})
 
 		// Building code view blocks with line number on server side.
 		escapedContent := &bytes.Buffer{}
diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go
index 00fd47b650..af3021da11 100644
--- a/routers/web/repo/view.go
+++ b/routers/web/repo/view.go
@@ -303,7 +303,7 @@ func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.Tr
 		return
 	}
 
-	rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
+	rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc), charset.ConvertOpts{})
 
 	if markupType := markup.Type(readmeFile.Name()); markupType != "" {
 		ctx.Data["IsMarkup"] = true
@@ -492,7 +492,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) {
 			break
 		}
 
-		rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
+		rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc), charset.ConvertOpts{})
 
 		shouldRenderSource := ctx.FormString("display") == "source"
 		readmeExist := util.IsReadmeFileName(blob.Name())