summaryrefslogtreecommitdiffstats
path: root/src/main/java/com/gitblit/wicket/MarkupProcessor.java
blob: 7facc59e35cf5a5988b09c306d02aa87f66df010 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
plugins:
  - stylelint-declaration-strict-value
  - stylelint-declaration-block-no-ignored-properties
  - stylelint-stylistic

ignoreFiles:
  - "**/*.go"

overrides:
  - files: ["**/chroma/*", "**/codemirror/*", "**/standalone/*", "**/console.css", "font_i18n.css"]
    rules:
      scale-unlimited/declaration-strict-value: null
  - files: ["**/chroma/*", "**/codemirror/*"]
    rules:
      block-no-empty: null
  - files: ["**/*.vue"]
    customSyntax: postcss-html

rules:
  alpha-value-notation: null
  annotation-no-unknown: true
  at-rule-allowed-list: null
  at-rule-disallowed-list: null
  at-rule-empty-line-before: null
  at-rule-no-unknown<
/*
 * Copyright 2013 gitblit.com.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.gitblit.wicket;

import static org.pegdown.FastEncoder.encode;

import java.io.Serializable;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.wicket.Page;
import org.apache.wicket.RequestCycle;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.mylyn.wikitext.confluence.core.ConfluenceLanguage;
import org.eclipse.mylyn.wikitext.core.parser.Attributes;
import org.eclipse.mylyn.wikitext.core.parser.MarkupParser;
import org.eclipse.mylyn.wikitext.core.parser.builder.HtmlDocumentBuilder;
import org.eclipse.mylyn.wikitext.core.parser.markup.MarkupLanguage;
import org.eclipse.mylyn.wikitext.mediawiki.core.MediaWikiLanguage;
import org.eclipse.mylyn.wikitext.textile.core.TextileLanguage;
import org.eclipse.mylyn.wikitext.tracwiki.core.TracWikiLanguage;
import org.eclipse.mylyn.wikitext.twiki.core.TWikiLanguage;
import org.pegdown.DefaultVerbatimSerializer;
import org.pegdown.LinkRenderer;
import org.pegdown.ToHtmlSerializer;
import org.pegdown.VerbatimSerializer;
import org.pegdown.ast.ExpImageNode;
import org.pegdown.ast.ExpLinkNode;
import org.pegdown.ast.RefImageNode;
import org.pegdown.ast.WikiLinkNode;
import org.pegdown.plugins.ToHtmlSerializerPlugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.IStoredSettings;
import com.gitblit.Keys;
import com.gitblit.models.PathModel;
import com.gitblit.servlet.RawServlet;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.MarkdownUtils;
import com.gitblit.utils.StringUtils;
import com.gitblit.utils.XssFilter;
import com.gitblit.wicket.pages.DocPage;
import com.google.common.base.Joiner;

/**
 * Processes markup content and generates html with repository-relative page and
 * image linking.
 *
 * @author James Moger
 *
 */
public class MarkupProcessor {

	public enum MarkupSyntax {
		PLAIN, MARKDOWN, TWIKI, TRACWIKI, TEXTILE, MEDIAWIKI, CONFLUENCE
	}

	private Logger logger = LoggerFactory.getLogger(getClass());

	private final IStoredSettings settings;

	private final XssFilter xssFilter;

	public static List<String> getMarkupExtensions(IStoredSettings settings) {
		List<String> list = new ArrayList<String>();
		list.addAll(settings.getStrings(Keys.web.confluenceExtensions));
		list.addAll(settings.getStrings(Keys.web.markdownExtensions));
		list.addAll(settings.getStrings(Keys.web.mediawikiExtensions));
		list.addAll(settings.getStrings(Keys.web.textileExtensions));
		list.addAll(settings.getStrings(Keys.web.tracwikiExtensions));
		list.addAll(settings.getStrings(Keys.web.twikiExtensions));
		return list;
	}

	public MarkupProcessor(IStoredSettings settings, XssFilter xssFilter) {
		this.settings = settings;
		this.xssFilter = xssFilter;
	}

	public List<String> getMarkupExtensions() {
		return getMarkupExtensions(settings);
	}

	public List<String> getAllExtensions() {
		List<String> list = getMarkupExtensions(settings);
		list.add("txt");
		list.add("TXT");
		return list;
	}

	private List<String> getRoots() {
		return settings.getStrings(Keys.web.documents);
	}

	private String [] getEncodings() {
		return settings.getStrings(Keys.web.blobEncodings).toArray(new String[0]);
	}

	private MarkupSyntax determineSyntax(String documentPath) {
		String ext = StringUtils.getFileExtension(documentPath).toLowerCase();
		if (StringUtils.isEmpty(ext)) {
			return MarkupSyntax.PLAIN;
		}

		if (settings.getStrings(Keys.web.confluenceExtensions).contains(ext)) {
			return MarkupSyntax.CONFLUENCE;
		} else if (settings.getStrings(Keys.web.markdownExtensions).contains(ext)) {
			return MarkupSyntax.MARKDOWN;
		} else if (settings.getStrings(Keys.web.mediawikiExtensions).contains(ext)) {
			return MarkupSyntax.MEDIAWIKI;
		} else if (settings.getStrings(Keys.web.textileExtensions).contains(ext)) {
			return MarkupSyntax.TEXTILE;
		} else if (settings.getStrings(Keys.web.tracwikiExtensions).contains(ext)) {
			return MarkupSyntax.TRACWIKI;
		} else if (settings.getStrings(Keys.web.twikiExtensions).contains(ext)) {
			return MarkupSyntax.TWIKI;
		}

		return MarkupSyntax.PLAIN;
	}

	public boolean hasRootDocs(Repository r) {
		List<String> roots = getRoots();
		List<String> extensions = getAllExtensions();
		List<PathModel> paths = JGitUtils.getFilesInPath(r, null, null);
		for (PathModel path : paths) {
			if (!path.isTree()) {
				String ext = StringUtils.getFileExtension(path.name).toLowerCase();
				String name = StringUtils.stripFileExtension(path.name).toLowerCase();

				if (roots.contains(name)) {
					if (StringUtils.isEmpty(ext) || extensions.contains(ext)) {
						return true;
					}
				}
			}
		}
		return false;
	}

	public List<MarkupDocument> getRootDocs(Repository r, String repositoryName, String commitId) {
		List<String> roots = getRoots();
		List<MarkupDocument> list = getDocs(r, repositoryName, commitId, roots);
		return list;
	}

	public MarkupDocument getReadme(Repository r, String repositoryName, String commitId) {
		List<MarkupDocument> list = getDocs(r, repositoryName, commitId, Arrays.asList("readme"));
		if (list.isEmpty()) {
			return null;
		}
		return list.get(0);
	}

	private List<MarkupDocument> getDocs(Repository r, String repositoryName, String commitId, List<String> names) {
		List<String> extensions = getAllExtensions();
		String [] encodings = getEncodings();
		Map<String, MarkupDocument> map = new HashMap<String, MarkupDocument>();
		RevCommit commit = JGitUtils.getCommit(r, commitId);
		List<PathModel> paths = JGitUtils.getFilesInPath(r, null, commit);
		for (PathModel path : paths) {
			if (!path.isTree()) {
				String ext = StringUtils.getFileExtension(path.name).toLowerCase();
				String name = StringUtils.stripFileExtension(path.name).toLowerCase();

				if (names.contains(name)) {
					if (StringUtils.isEmpty(ext) || extensions.contains(ext)) {
						String markup = JGitUtils.getStringContent(r, commit.getTree(), path.name, encodings);
						MarkupDocument doc = parse(repositoryName, commitId, path.name, markup);
						map.put(name, doc);
					}
				}
			}
		}
		// return document list in requested order
		List<MarkupDocument> list = new ArrayList<MarkupDocument>();
		for (String name : names) {
			if (map.containsKey(name)) {
				list.add(map.get(name));
			}
		}
		return list;
	}

	public MarkupDocument parse(String repositoryName, String commitId, String documentPath, String markupText) {
		final MarkupSyntax syntax = determineSyntax(documentPath);
		final MarkupDocument doc = new MarkupDocument(documentPath, markupText, syntax);

		if (markupText != null) {
			try {
				switch (syntax){
				case CONFLUENCE:
					parse(doc, repositoryName, commitId, new ConfluenceLanguage());
					break;
				case MARKDOWN:
					parse(doc, repositoryName, commitId);
					break;
				case MEDIAWIKI:
					parse(doc, repositoryName, commitId, new MediaWikiLanguage());
					break;
				case TEXTILE:
					parse(doc, repositoryName, commitId, new TextileLanguage());
					break;
				case TRACWIKI:
					parse(doc, repositoryName, commitId, new TracWikiLanguage());
					break;
				case TWIKI:
					parse(doc, repositoryName, commitId, new TWikiLanguage());
					break;
				default:
					doc.html = MarkdownUtils.transformPlainText(markupText);
					break;
				}
			} catch (Exception e) {
				logger.error("failed to transform " + syntax, e);
			}
		}

		if (doc.html == null) {
			// failed to transform markup
			if (markupText == null) {
				markupText = String.format("Document <b>%1$s</b> not found in <em>%2$s</em>", documentPath, repositoryName);
			}
			markupText = MessageFormat.format("<div class=\"alert alert-error\"><strong>{0}:</strong> {1}</div>{2}", "Error", "failed to parse markup", markupText);
			doc.html = StringUtils.breakLinesForHtml(markupText);
		}

		return doc;
	}

	/**
	 * Parses the markup using the specified markup language
	 *
	 * @param doc
	 * @param repositoryName
	 * @param commitId
	 * @param lang
	 */
	private void parse(final MarkupDocument doc, final String repositoryName, final String commitId, MarkupLanguage lang) {
		StringWriter writer = new StringWriter();
		HtmlDocumentBuilder builder = new HtmlDocumentBuilder(writer) {

			@Override
			public void image(Attributes attributes, String imagePath) {
				String url;
				if (imagePath.indexOf("://") == -1) {
					// relative image
					String path = doc.getRelativePath(imagePath);
					String contextUrl = RequestCycle.get().getRequest().getRelativePathPrefixToContextRoot();
					url = RawServlet.asLink(contextUrl, repositoryName, commitId, path);
				} else {
					// absolute image
					url = imagePath;
				}
				super.image(attributes, url);
			}

			@Override
			public void link(Attributes attributes, String hrefOrHashName, String text) {
				String url;
				if (hrefOrHashName.charAt(0) != '#') {
					if (hrefOrHashName.indexOf("://") == -1) {
						// relative link
						String path = doc.getRelativePath(hrefOrHashName);
						url = getWicketUrl(DocPage.class, repositoryName, commitId, path);
					} else {
						// absolute link
						url = hrefOrHashName;
					}
				} else {
					// page-relative hash link
					url = hrefOrHashName;
				}
				super.link(attributes, url, text);
			}
		};

		// avoid the <html> and <body> tags
		builder.setEmitAsDocument(false);

		MarkupParser parser = new MarkupParser(lang);
		parser.setBuilder(builder);
		parser.parse(doc.markup);

		final String content = writer.toString();
		final String safeContent = xssFilter.relaxed(content);

		doc.html = safeContent;
	}

	/**
	 * Parses the document as Markdown using Pegdown.
	 *
	 * @param doc
	 * @param repositoryName
	 * @param commitId
	 */
	private void parse(final MarkupDocument doc, final String repositoryName, final String commitId) {
		LinkRenderer renderer = new LinkRenderer() {

			@Override
			public Rendering render(ExpImageNode node, String text) {
				if (node.url.indexOf("://") == -1) {
					// repository-relative image link
					String path = doc.getRelativePath(node.url);
					String contextUrl = RequestCycle.get().getRequest().getRelativePathPrefixToContextRoot();
					String url = RawServlet.asLink(contextUrl, repositoryName, commitId, path);
					return new Rendering(url, text);
				}
				// absolute image link
				return new Rendering(node.url, text);
			}

			@Override
			public Rendering render(RefImageNode node, String url, String title, String alt) {
				Rendering rendering;
				if (url.indexOf("://") == -1) {
					// repository-relative image link
					String path = doc.getRelativePath(url);
					String contextUrl = RequestCycle.get().getRequest().getRelativePathPrefixToContextRoot();
					String wurl = RawServlet.asLink(contextUrl, repositoryName, commitId, path);
					rendering = new Rendering(wurl, alt);
				} else {
					// absolute image link
					rendering = new Rendering(url, alt);
				}
				return StringUtils.isEmpty(title) ? rendering : rendering.withAttribute("title", encode(title));
			}

			@Override
			public Rendering render(WikiLinkNode node) {
				String path = doc.getRelativePath(node.getText());
				String name = getDocumentName(path);
				String url = getWicketUrl(DocPage.class, repositoryName, commitId, path);
				return new Rendering(url, name);
			}

			@Override
			public Rendering render(ExpLinkNode node, String text) {
				String path = doc.getRelativePath(node.url);
				String url = getWicketUrl(DocPage.class, repositoryName, commitId, path);
				return new Rendering(url, text);
			}
		};

		final String content = MarkdownUtils.transformMarkdown(doc.markup, renderer);
		final String safeContent = xssFilter.relaxed(content);

		doc.html = safeContent;
	}

	private String getWicketUrl(Class<? extends Page> pageClass, final String repositoryName, final String commitId, final String document) {
		String fsc = settings.getString(Keys.web.forwardSlashCharacter, "/");
		String encodedPath = document.replace(' ', '-');
		try {
			encodedPath = URLEncoder.encode(encodedPath, "UTF-8");
		} catch (UnsupportedEncodingException e) {
			logger.error(null, e);
		}
		encodedPath = encodedPath.replace("/", fsc).replace("%2F", fsc);

		String url = RequestCycle.get().urlFor(pageClass, WicketUtils.newPathParameter(repositoryName, commitId, encodedPath)).toString();
		return url;
	}

	private String getDocumentName(final String document) {
		// extract document name
		String name = StringUtils.stripFileExtension(document);
		name = name.replace('_', ' ');
		if (name.indexOf('/') > -1) {
			name = name.substring(name.lastIndexOf('/') + 1);
		}
		return name;
	}

	public static class MarkupDocument implements Serializable {

		private static final long serialVersionUID = 1L;

		public final String documentPath;
		public final String markup;
		public final MarkupSyntax syntax;
		public String html;

		MarkupDocument(String documentPath, String markup, MarkupSyntax syntax) {
			this.documentPath = documentPath;
			this.markup = markup;
			this.syntax = syntax;
		}

		String getCurrentPath() {
			String basePath = "";
			if (documentPath.indexOf('/') > -1) {
				basePath = documentPath.substring(0, documentPath.lastIndexOf('/') + 1);
				if (basePath.charAt(0) == '/') {
					return basePath.substring(1);
				}
			}
			return basePath;
		}

		String getRelativePath(String ref) {
			if (ref.charAt(0) == '/') {
				// absolute path in repository
				return ref.substring(1);
			} else {
				// resolve relative repository path
				String cp = getCurrentPath();
				if (StringUtils.isEmpty(cp)) {
					return ref;
				}
				// this is a simple relative path resolver
				List<String> currPathStrings = new ArrayList<String>(Arrays.asList(cp.split("/")));
				String file = ref;
				while (file.startsWith("../")) {
					// strip ../ from the file reference
					// drop the last path element
					file = file.substring(3);
					currPathStrings.remove(currPathStrings.size() - 1);
				}
				currPathStrings.add(file);
				String path = Joiner.on("/").join(currPathStrings);
				return path;
			}
		}
	}

	/**
	 * This class implements a workaround for a bug reported in issue-379.
	 * The bug was introduced by my own pegdown pull request #115.
	 *
	 * @author James Moger
	 *
	 */
	public static class WorkaroundHtmlSerializer extends ToHtmlSerializer {

		 public WorkaroundHtmlSerializer(final LinkRenderer linkRenderer) {
			 super(linkRenderer,
					 Collections.<String, VerbatimSerializer>singletonMap(VerbatimSerializer.DEFAULT, DefaultVerbatimSerializer.INSTANCE),
					 Collections.<ToHtmlSerializerPlugin>emptyList());
		    }
	    private void printAttribute(String name, String value) {
	        printer.print(' ').print(name).print('=').print('"').print(value).print('"');
	    }

	    /* Reimplement print image tag to eliminate a trailing double-quote */
		@Override
	    protected void printImageTag(LinkRenderer.Rendering rendering) {
	        printer.print("<img");
	        printAttribute("src", rendering.href);
	        printAttribute("alt", rendering.text);
	        for (LinkRenderer.Attribute attr : rendering.attributes) {
	            printAttribute(attr.name, attr.value);
	        }
	        printer.print("/>");
	    }
	}
}