Convert EOL to UNIX-style to render MD properly (#8925)

* Convert EOL to UNIX-style to render MD properly

* Update modules/markup/markdown/markdown.go

Co-Authored-By: zeripath <art27@cantab.net>

* Fix lint optimization

* Check for empty content before conversion

* Update modules/util/util.go

Co-Authored-By: zeripath <art27@cantab.net>

* Improved checks and tests

* Add paragraph render test

* Improve speed even more, improve tests

* Small improvement by @gary-kim

* Fix test for DOS

* More improvements

* Restart CI

1 files changed, 59 insertions, 0 deletions

diff --git a/modules/util/util_test.go b/modules/util/util_test.go
index 2475065059..04ab42f292 100644
--- a/modules/util/util_test.go
+++ b/modules/util/util_test.go
@@ -5,6 +5,7 @@
 package util
 
 import (
+	"strings"
 	"testing"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -94,3 +95,61 @@ func TestIsEmptyString(t *testing.T) {
 		assert.Equal(t, v.expected, IsEmptyString(v.s))
 	}
 }
+
+func Test_NormalizeEOL(t *testing.T) {
+	data1 := []string{
+		"",
+		"This text starts with empty lines",
+		"another",
+		"",
+		"",
+		"",
+		"Some other empty lines in the middle",
+		"more.",
+		"And more.",
+		"Ends with empty lines too.",
+		"",
+		"",
+		"",
+	}
+
+	data2 := []string{
+		"This text does not start with empty lines",
+		"another",
+		"",
+		"",
+		"",
+		"Some other empty lines in the middle",
+		"more.",
+		"And more.",
+		"Ends without EOLtoo.",
+	}
+
+	buildEOLData := func(data []string, eol string) []byte {
+		return []byte(strings.Join(data, eol))
+	}
+
+	dos := buildEOLData(data1, "\r\n")
+	unix := buildEOLData(data1, "\n")
+	mac := buildEOLData(data1, "\r")
+
+	assert.Equal(t, unix, NormalizeEOL(dos))
+	assert.Equal(t, unix, NormalizeEOL(mac))
+	assert.Equal(t, unix, NormalizeEOL(unix))
+
+	dos = buildEOLData(data2, "\r\n")
+	unix = buildEOLData(data2, "\n")
+	mac = buildEOLData(data2, "\r")
+
+	assert.Equal(t, unix, NormalizeEOL(dos))
+	assert.Equal(t, unix, NormalizeEOL(mac))
+	assert.Equal(t, unix, NormalizeEOL(unix))
+
+	assert.Equal(t, []byte("one liner"), NormalizeEOL([]byte("one liner")))
+	assert.Equal(t, []byte("\n"), NormalizeEOL([]byte("\n")))
+	assert.Equal(t, []byte("\ntwo liner"), NormalizeEOL([]byte("\ntwo liner")))
+	assert.Equal(t, []byte("two liner\n"), NormalizeEOL([]byte("two liner\n")))
+	assert.Equal(t, []byte{}, NormalizeEOL([]byte{}))
+
+	assert.Equal(t, []byte("mix\nand\nmatch\n."), NormalizeEOL([]byte("mix\r\nand\rmatch\n.")))
+}