Add warning for BIDI characters in page renders and in diffs (#17562)

Fix #17514

Given the comments I've adjusted this somewhat. The numbers of characters detected are increased and include things like the use of U+300 to make à instead of à and non-breaking spaces.

There is a button which can be used to escape the content to show it.

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: Gwyneth Morgan <gwymor@tilde.club>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

4 files changed, 98 insertions, 13 deletions

diff --git a/templates/repo/diff/blob_excerpt.tmpl b/templates/repo/diff/blob_excerpt.tmpl
index 792c539ac5..e529ed3bcd 100644
--- a/templates/repo/diff/blob_excerpt.tmpl
+++ b/templates/repo/diff/blob_excerpt.tmpl
@@ -19,14 +19,26 @@
 					</a>
 				{{end}}
 			</td>
-			<td colspan="5" class="lines-code lines-code-old "><code class="code-inner">{{$.section.GetComputedInlineDiffFor $line}}</code></td>
+			<td colspan="5" class="lines-code lines-code-old ">{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}{{$inlineDiff}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code></td>
 		{{else}}
 			<td class="lines-num lines-num-old" data-line-num="{{if $line.LeftIdx}}{{$line.LeftIdx}}{{end}}"><span rel="{{if $line.LeftIdx}}diff-{{Sha1 $.fileName}}L{{$line.LeftIdx}}{{end}}"></span></td>
 			<td class="blob-excerpt lines-type-marker lines-type-marker-old">{{if $line.LeftIdx}}<span class="mono" data-type-marker=""></span>{{end}}</td>
-			<td class="blob-excerpt lines-code lines-code-old halfwidth"><code class="code-inner">{{if $line.LeftIdx}}{{$.section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+			<td class="blob-excerpt lines-code lines-code-old halfwidth">{{/*
+				*/}}{{if $line.LeftIdx}}{{/*
+						*/}}{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+					*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+					*/}}{{end}}{{/*
+				*/}}</td>
 			<td class="lines-num lines-num-new" data-line-num="{{if $line.RightIdx}}{{$line.RightIdx}}{{end}}"><span rel="{{if $line.RightIdx}}diff-{{Sha1 $.fileName}}R{{$line.RightIdx}}{{end}}"></span></td>
 			<td class="blob-excerpt lines-type-marker lines-type-marker-new">{{if $line.RightIdx}}<span class="mono" data-type-marker=""></span>{{end}}</td>
-			<td class="blob-excerpt lines-code lines-code-new halfwidth"><code class="code-inner">{{if $line.RightIdx}}{{$.section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+			<td class="blob-excerpt lines-code lines-code-new halfwidth">{{/*
+				*/}}{{if $line.RightIdx}}{{/*
+					*/}}{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+				*/}}{{else}}{{/*
+					*/}}<code class="code-inner"></code>{{/*
+				*/}}{{end}}{{/*
+			*/}}</td>
 		{{end}}
 	</tr>
 	{{end}}
diff --git a/templates/repo/diff/box.tmpl b/templates/repo/diff/box.tmpl
index 3ab7a11bbd..f115a5f499 100644
--- a/templates/repo/diff/box.tmpl
+++ b/templates/repo/diff/box.tmpl
@@ -94,6 +94,10 @@
 							{{if $file.IsProtected}}
 								<span class="ui basic label">{{$.i18n.Tr "repo.diff.protected"}}</span>
 							{{end}}
+							{{if not (or $file.IsIncomplete $file.IsBin $file.IsSubmodule)}}
+								<a class="ui basic tiny button unescape-button">{{$.i18n.Tr "repo.unescape_control_characters"}}</a>
+								<a class="ui basic tiny button escape-button" style="display: none;">{{$.i18n.Tr "repo.escape_control_characters"}}</a>
+							{{end}}
 							{{if and (not $file.IsSubmodule) (not $.PageIsWiki)}}
 								{{if $file.IsDeleted}}
 									<a class="ui basic tiny button" rel="nofollow" href="{{$.BeforeSourcePath}}/{{PathEscapeSegments .Name}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
@@ -104,7 +108,7 @@
 						</div>
 					</h4>
 					<div class="diff-file-body ui attached unstackable table segment">
-						<div id="diff-source-{{$i}}" class="file-body file-code code-diff{{if $.IsSplitStyle}} code-diff-split{{else}} code-diff-unified{{end}}{{if $showFileViewToggle}} hide{{end}}">
+						<div id="diff-source-{{$i}}" class="file-body file-code unicode-escaped code-diff{{if $.IsSplitStyle}} code-diff-split{{else}} code-diff-unified{{end}}{{if $showFileViewToggle}} hide{{end}}">
 							{{if or $file.IsIncomplete $file.IsBin}}
 								<div class="diff-file-body binary" style="padding: 5px 10px;">
 									{{if $file.IsIncomplete}}
diff --git a/templates/repo/diff/section_split.tmpl b/templates/repo/diff/section_split.tmpl
index 81223642db..754f7cec10 100644
--- a/templates/repo/diff/section_split.tmpl
+++ b/templates/repo/diff/section_split.tmpl
@@ -21,23 +21,75 @@
 								{{svg "octicon-fold"}}
 							</a>
 						{{end}}
-					</td>
-					<td colspan="5" class="lines-code lines-code-old "><code class="code-inner">{{$section.GetComputedInlineDiffFor $line}}</span></td>
+					</td>{{$inlineDiff := $section.GetComputedInlineDiffFor $line}}
+					<td class="lines-escape lines-escape-old">{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}</td>
+					<td colspan="6" class="lines-code lines-code-old "><code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</span></td>
 				{{else if and (eq .GetType 3) $hasmatch}}{{/* DEL */}}
 					{{$match := index $section.Lines $line.Match}}
+					{{- $leftDiff := ""}}{{if $line.LeftIdx}}{{$leftDiff = $section.GetComputedInlineDiffFor $line}}{{end}}
+					{{- $rightDiff := ""}}{{if $match.RightIdx}}{{$rightDiff = $section.GetComputedInlineDiffFor $match}}{{end}}
 					<td class="lines-num lines-num-old del-code" data-line-num="{{$line.LeftIdx}}"><span rel="diff-{{Sha1 $file.Name}}L{{$line.LeftIdx}}"></span></td>
+					<td class="lines-escape del-code lines-escape-old">{{if $line.LeftIdx}}{{if $leftDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-old del-code"><span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span></td>
-					<td class="lines-code lines-code-old halfwidth del-code">{{if and $.root.SignedUserID $.root.PageIsPullFiles}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $line.LeftIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+					<td class="lines-code lines-code-old halfwidth del-code">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $line.LeftIdx}}{{/*
+							*/}}<code {{if $leftDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$leftDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 					<td class="lines-num lines-num-new add-code" data-line-num="{{if $match.RightIdx}}{{$match.RightIdx}}{{end}}"><span rel="{{if $match.RightIdx}}diff-{{Sha1 $file.Name}}R{{$match.RightIdx}}{{end}}"></span></td>
+					<td class="lines-escape add-code lines-escape-new">{{if $match.RightIdx}}{{if $rightDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-new add-code">{{if $match.RightIdx}}<span class="mono" data-type-marker="{{$match.GetLineTypeMarker}}"></span>{{end}}</td>
-					<td class="lines-code lines-code-new halfwidth add-code">{{if and $.root.SignedUserID $.root.PageIsPullFiles}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $match.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$match.RightIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $match.RightIdx}}{{$section.GetComputedInlineDiffFor $match}}{{end}}</code></td>
+					<td class="lines-code lines-code-new halfwidth add-code">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $match.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$match.RightIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $match.RightIdx}}{{/*
+							*/}}<code {{if $rightDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$rightDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+							*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 				{{else}}
+					{{$inlineDiff := $section.GetComputedInlineDiffFor $line}}
 					<td class="lines-num lines-num-old" data-line-num="{{if $line.LeftIdx}}{{$line.LeftIdx}}{{end}}"><span rel="{{if $line.LeftIdx}}diff-{{Sha1 $file.Name}}L{{$line.LeftIdx}}{{end}}"></span></td>
+					<td class="lines-escape lines-escape-old">{{if $line.LeftIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-old">{{if $line.LeftIdx}}<span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span>{{end}}</td>
-					<td class="lines-code lines-code-old halfwidth">{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 2))}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $line.LeftIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+					<td class="lines-code lines-code-old halfwidth">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 2))}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $line.LeftIdx}}{{/*
+							*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 					<td class="lines-num lines-num-new" data-line-num="{{if $line.RightIdx}}{{$line.RightIdx}}{{end}}"><span rel="{{if $line.RightIdx}}diff-{{Sha1 $file.Name}}R{{$line.RightIdx}}{{end}}"></span></td>
+					<td class="lines-escape lines-escape-new">{{if $line.RightIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-new">{{if $line.RightIdx}}<span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span>{{end}}</td>
-					<td class="lines-code lines-code-new halfwidth">{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 3))}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $line.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$line.RightIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $line.RightIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+					<td class="lines-code lines-code-new halfwidth">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 3))}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $line.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$line.RightIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $line.RightIdx}}{{/*
+							*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 				{{end}}
 			</tr>
 			{{if and (eq .GetType 3) $hasmatch}}
@@ -45,6 +97,7 @@
 				{{if or (gt (len $line.Comments) 0) (gt (len $match.Comments) 0)}}
 					<tr class="add-comment" data-line-type="{{DiffLineTypeToStr .GetType}}">
 						<td class="lines-num"></td>
+						<td class="lines-escape"></td>
 						<td class="lines-type-marker"></td>
 						<td class="add-comment-left">
 							{{if gt (len $line.Comments) 0}}
@@ -59,6 +112,7 @@
 							{{end}}
 						</td>
 						<td class="lines-num"></td>
+						<td class="lines-escape"></td>
 						<td class="lines-type-marker"></td>
 						<td class="add-comment-right">
 							{{if eq $line.GetCommentSide "proposed"}}
@@ -75,6 +129,7 @@
 			{{else if gt (len $line.Comments) 0}}
 				<tr class="add-comment" data-line-type="{{DiffLineTypeToStr .GetType}}">
 					<td class="lines-num"></td>
+					<td class="lines-escape"></td>
 					<td class="lines-type-marker"></td>
 					<td class="add-comment-left">
 						{{if gt (len $line.Comments) 0}}
@@ -84,6 +139,7 @@
 						{{end}}
 					</td>
 					<td class="lines-num"></td>
+					<td class="lines-escape"></td>
 					<td class="lines-type-marker"></td>
 					<td class="add-comment-right">
 						{{if eq $line.GetCommentSide "proposed"}}
diff --git a/templates/repo/diff/section_unified.tmpl b/templates/repo/diff/section_unified.tmpl
index 74634a760f..93f9af52b4 100644
--- a/templates/repo/diff/section_unified.tmpl
+++ b/templates/repo/diff/section_unified.tmpl
@@ -25,16 +25,29 @@
 					<td class="lines-num lines-num-old" data-line-num="{{if $line.LeftIdx}}{{$line.LeftIdx}}{{end}}"><span rel="{{if $line.LeftIdx}}diff-{{Sha1 $file.Name}}L{{$line.LeftIdx}}{{end}}"></span></td>
 					<td class="lines-num lines-num-new" data-line-num="{{if $line.RightIdx}}{{$line.RightIdx}}{{end}}"><span rel="{{if $line.RightIdx}}diff-{{Sha1 $file.Name}}R{{$line.RightIdx}}{{end}}"></span></td>
 				{{end}}
+				{{$inlineDiff := $section.GetComputedInlineDiffFor $line -}}
+				<td class="lines-escape">{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}</td>
 				<td class="lines-type-marker"><span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span></td>
 				{{if eq .GetType 4}}
-					<td class="chroma lines-code blob-hunk"><code class="code-inner">{{$section.GetComputedInlineDiffFor $line}}</code></td>
+					<td class="chroma lines-code blob-hunk">{{/*
+						*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+					*/}}
+				{{$line.Content}}
+					</td>
 				{{else}}
-					<td class="chroma lines-code{{if (not $line.RightIdx)}} lines-code-old{{end}}">{{if and $.root.SignedUserID $.root.PageIsPullFiles}}<a class="ui primary button add-code-comment add-code-comment-{{if $line.RightIdx}}right{{else}}left{{end}}{{if (not $line.CanComment)}} invisible{{end}}" data-side="{{if $line.RightIdx}}right{{else}}left{{end}}" data-idx="{{if $line.RightIdx}}{{$line.RightIdx}}{{else}}{{$line.LeftIdx}}{{end}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{$section.GetComputedInlineDiffFor $line}}</code></td>
+					<td class="chroma lines-code{{if (not $line.RightIdx)}} lines-code-old{{end}}">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-{{if $line.RightIdx}}right{{else}}left{{end}}{{if (not $line.CanComment)}} invisible{{end}}" data-side="{{if $line.RightIdx}}right{{else}}left{{end}}" data-idx="{{if $line.RightIdx}}{{$line.RightIdx}}{{else}}{{$line.LeftIdx}}{{end}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+					*/}}</td>
 				{{end}}
 			</tr>
 			{{if gt (len $line.Comments) 0}}
 				<tr class="add-comment" data-line-type="{{DiffLineTypeToStr .GetType}}">
-					<td colspan="2" class="lines-num"></td>
+					<td colspan="3" class="lines-num"></td>
 					<td class="add-comment-left add-comment-right" colspan="2">
 						{{template "repo/diff/conversation" mergeinto $.root "comments" $line.Comments}}
 					</td>