Rewrite reference processing code in preparation for opening/closing from comment references (#8261)

* Add a markdown stripper for mentions and xrefs

* Improve comments

* Small code simplification

* Move reference code to modules/references

* Fix typo

* Make MarkdownStripper return [][]byte

* Implement preliminary keywords parsing

* Add FIXME comment

* Fix comment

* make fmt

* Fix permissions check

* Fix text assumptions

* Fix imports

* Fix lint, fmt

* Fix unused import

* Add missing export comment

* Bypass revive on implemented interface

* Move mdstripper into its own package

* Support alphanumeric patterns

* Refactor FindAllMentions

* Move mentions test to references

* Parse mentions from reference package

* Refactor code to implement renderizable references

* Fix typo

* Move patterns and tests to the references package

* Fix nil reference

* Preliminary rendering attempt of closing keywords

* Normalize names, comments, general tidy-up

* Add CSS style for action keywords

* Fix permission for admin and owner

* Fix golangci-lint

* Fix golangci-lint

1 files changed, 71 insertions, 0 deletions

diff --git a/modules/markup/mdstripper/mdstripper_test.go b/modules/markup/mdstripper/mdstripper_test.go
new file mode 100644
index 0000000000..157fe1975b
--- /dev/null
+++ b/modules/markup/mdstripper/mdstripper_test.go
@@ -0,0 +1,71 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mdstripper
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarkdownStripper(t *testing.T) {
+	type testItem struct {
+		markdown      string
+		expectedText  []string
+		expectedLinks []string
+	}
+
+	list := []testItem{
+		{
+			`
+## This is a title
+
+This is [one](link) to paradise.
+This **is emphasized**.
+This: should coallesce.
+
+` + "```" + `
+This is a code block.
+This should not appear in the output at all.
+` + "```" + `
+
+* Bullet 1
+* Bullet 2
+
+A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE.
+		`,
+			[]string{
+				"This is a title",
+				"This is",
+				"to paradise.",
+				"This",
+				"is emphasized",
+				".",
+				"This: should coallesce.",
+				"Bullet 1",
+				"Bullet 2",
+				"A HIDDEN",
+				"IN THIS LINE.",
+			},
+			[]string{
+				"link",
+			}},
+	}
+
+	for _, test := range list {
+		text, links := StripMarkdown([]byte(test.markdown))
+		rawlines := strings.Split(text, "\n")
+		lines := make([]string, 0, len(rawlines))
+		for _, line := range rawlines {
+			line := strings.TrimSpace(line)
+			if line != "" {
+				lines = append(lines, line)
+			}
+		}
+		assert.EqualValues(t, test.expectedText, lines)
+		assert.EqualValues(t, test.expectedLinks, links)
+	}
+}