mirrors
/
rspamd
mirror of https://github.com/vstakhov/rspamd.git
Watch 1
Star 0
Fork 0
Code Issues 0 Releases 159 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
21213 Commits
28 Branches
Branch: master
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'master'
${ noResults }
rspamd/src/libserver/html/html_tests.cxx

html_tests.cxx 11KB
Raw Permalink Blame History

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304 
  1. /*

  2.  * Copyright 2024 Vsevolod Stakhov

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  *    http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "config.h"

  18. #include "html.hxx"

  19. #include "libserver/task.h"

  20. 

  21. #include <vector>

  22. #include <fmt/core.h>

  23. 

  24. 

  25. #define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL

  26. #include "doctest/doctest.h"

  27. 

  28. namespace rspamd::html {

  29. 

  30. /*

  31.  * Tests part

  32.  */

  33. 

  34. TEST_SUITE("html")

  35. {

  36. 	TEST_CASE("html parsing")

  37. 	{

  38. 

  39. 		const std::vector<std::pair<std::string, std::string>> cases{

  40. 			{"<html><!DOCTYPE html><body>", "+html;++xml;++body;"},

  41. 			{"<html><div><div></div></div></html>", "+html;++div;+++div;"},

  42. 			{"<html><div><div></div></html>", "+html;++div;+++div;"},

  43. 			{"<html><div><div></div></html></div>", "+html;++div;+++div;"},

  44. 			{"<p><p><a></p></a></a>", "+p;++p;+++a;"},

  45. 			{"<div><a href=\"http://example.com\"></div></a>", "+div;++a;"},

  46. 			/* Broken, as I don't know how the hell this should be really parsed */

  47. 			//{"<html><!DOCTYPE html><body><head><body></body></html></body></html>",

  48. 			//												   "+html;++xml;++body;+++head;+++body;"}

  49. 		};

  50. 

  51. 		rspamd_url_init(NULL);

  52. 		auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),

  53. 										"html", 0);

  54. 		struct rspamd_task fake_task;

  55. 		memset(&fake_task, 0, sizeof(fake_task));

  56. 		fake_task.task_pool = pool;

  57. 

  58. 		for (const auto &c: cases) {

  59. 			SUBCASE((std::string("extract tags from: ") + c.first).c_str())

  60. 			{

  61. 				GByteArray *tmp = g_byte_array_sized_new(c.first.size());

  62. 				g_byte_array_append(tmp, (const uint8_t *) c.first.data(), c.first.size());

  63. 				auto *hc = html_process_input(&fake_task, tmp, nullptr, nullptr, nullptr, true, nullptr);

  64. 				CHECK(hc != nullptr);

  65. 				auto dump = html_debug_structure(*hc);

  66. 				CHECK(c.second == dump);

  67. 				g_byte_array_free(tmp, TRUE);

  68. 			}

  69. 		}

  70. 

  71. 		rspamd_mempool_delete(pool);

  72. 	}

  73. 

  74. 	TEST_CASE("html text extraction")

  75. 	{

  76. 		using namespace std::string_literals;

  77. 		const std::vector<std::pair<std::string, std::string>> cases{

  78. 			{"test", "test"},

  79. 			{"test\0"s, "test\uFFFD"s},

  80. 			{"test\0test"s, "test\uFFFDtest"s},

  81. 			{"test\0\0test"s, "test\uFFFD\uFFFDtest"s},

  82. 			{"test   ", "test"},

  83. 			{"test   foo,   bar", "test foo, bar"},

  84. 			{"<p>text</p>", "text\n"},

  85. 			{"olo<p>text</p>lolo", "olo\ntext\nlolo"},

  86. 			{"<div>foo</div><div>bar</div>", "foo\nbar\n"},

  87. 			{"<b>foo<i>bar</b>baz</i>", "foobarbaz"},

  88. 			{"<b>foo<i>bar</i>baz</b>", "foobarbaz"},

  89. 			{"foo<br>baz", "foo\nbaz"},

  90. 			{"<a href=https://example.com>test</a>", "test"},

  91. 			{"<img alt=test>", "test"},

  92. 			{"  <body>\n"

  93. 			 "    <!-- escape content -->\n"

  94. 			 "    a&nbsp;b a &gt; b a &lt; b a &amp; b &apos;a &quot;a&quot;\n"

  95. 			 "  </body>",

  96. 			 R"|(a b a > b a < b a & b 'a "a")|"},

  97. 			/* XML tags */

  98. 			{"<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"

  99. 			 " <!DOCTYPE html\n"

  100. 			 "   PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n"

  101. 			 "   \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"

  102. 			 "<body>test</body>",

  103. 			 "test"},

  104. 			{"<html><head><meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\"></head>"

  105. 			 "  <body>\n"

  106. 			 "    <p><br>\n"

  107. 			 "    </p>\n"

  108. 			 "    <div class=\"moz-forward-container\"><br>\n"

  109. 			 "      <br>\n"

  110. 			 "      test</div>"

  111. 			 "</body>",

  112. 			 "\n\n\ntest\n"},

  113. 			{"<div>fi<span style=\"FONT-SIZE: 0px\">le </span>"

  114. 			 "sh<span style=\"FONT-SIZE: 0px\">aring </span></div>",

  115. 			 "fish\n"},

  116. 			/* FIXME: broken until rework of css parser */

  117. 			//{"<div>fi<span style=\"FONT-SIZE: 0px\">le </span>"

  118. 			// "sh<span style=\"FONT-SIZE: 0px\">aring </div>foo</span>", "fish\nfoo"},

  119. 			/* Complex html with bad tags */

  120. 			{"<!DOCTYPE html>\n"

  121. 			 "<html lang=\"en\">\n"

  122. 			 "  <head>\n"

  123. 			 "    <meta charset=\"utf-8\">\n"

  124. 			 "    <title>title</title>\n"

  125. 			 "    <link rel=\"stylesheet\" href=\"style.css\">\n"

  126. 			 "    <script src=\"script.js\"></script>\n"

  127. 			 "  </head>\n"

  128. 			 "  <body>\n"

  129. 			 "    <!-- page content -->\n"

  130. 			 "    Hello, world! <b>test</b>\n"

  131. 			 "    <p>data<>\n"

  132. 			 "    </P>\n"

  133. 			 "    <b>stuff</p>?\n"

  134. 			 "  </body>\n"

  135. 			 "</html>",

  136. 			 "Hello, world! test \ndata<>\nstuff\n?"},

  137. 			{"<p><!--comment-->test</br></hr><br>", "test\n"},

  138. 			/* Tables */

  139. 			{"<table>\n"

  140. 			 "      <tr>\n"

  141. 			 "        <th>heada</th>\n"

  142. 			 "        <th>headb</th>\n"

  143. 			 "      </tr>\n"

  144. 			 "      <tr>\n"

  145. 			 "        <td>data1</td>\n"

  146. 			 "        <td>data2</td>\n"

  147. 			 "      </tr>\n"

  148. 			 "    </table>",

  149. 			 "heada headb\ndata1 data2\n"},

  150. 			/* Invalid closing br and hr + comment */

  151. 			{"  <body>\n"

  152. 			 "    <!-- page content -->\n"

  153. 			 "    Hello, world!<br>test</br><br>content</hr>more content<br>\n"

  154. 			 "    <div>\n"

  155. 			 "      content inside div\n"

  156. 			 "    </div>\n"

  157. 			 "  </body>",

  158. 			 "Hello, world!\ntest\ncontentmore content\ncontent inside div\n"},

  159. 			/* First closing tag */

  160. 			{"</head>\n"

  161. 			 "<body>\n"

  162. 			 "<p> Hello. I have some bad news.\n"

  163. 			 "<br /> <br /> <br /> <strong> <br /> <br /> <br /> <br /> <br /> <br /> <br /> <br /> </strong><span> <br /> </span>test</p>\n"

  164. 			 "</body>\n"

  165. 			 "</html>",

  166. 			 "Hello. I have some bad news. \n\n\n\n\n\n\n\n\n\n\n\ntest\n"},

  167. 			/* Invalid tags */

  168. 			{"lol <sht> omg </sht> oh my!\n"

  169. 			 "<name>words words</name> goodbye",

  170. 			 "lol omg oh my! words words goodbye"},

  171. 			/* Invisible stuff */

  172. 			{"<div style=\"color:#555555;font-family:Arial, 'Helvetica Neue', Helvetica, sans-serif;line-height:1.2;padding-top:10px;padding-right:10px;padding-bottom:10px;padding-left:10px;font-style: italic;\">\n"

  173. 			 "<p style=\"font-size: 11px; line-height: 1.2; color: #555555; font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; mso-line-height-alt: 14px; margin: 0;\">\n"

  174. 			 "<span style=\"color:#FFFFFF; \">F</span>Sincerely,</p>\n"

  175. 			 "<p style=\"font-size: 11px; line-height: 1.2; color: #555555; font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; mso-line-height-alt: 14px; margin: 0;\">\n"

  176. 			 "<span style=\"color:#FFFFFF; \">8</span>Sky<span style=\"opacity:1;\"></span>pe<span style=\"color:#FFFFFF; \">F</span>Web<span style=\"color:#FFFFFF; \">F</span></p>\n"

  177. 			 "<span style=\"color:#FFFFFF; \">kreyes</span>\n"

  178. 			 "<p style=\"font-size: 11px; line-height: 1.2; color: #555555; font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; mso-line-height-alt: 14px; margin: 0;\">\n"

  179. 			 "&nbsp;</p>",

  180. 			 " Sincerely,\n Skype Web\n"},

  181. 			{"lala<p hidden>fafa</p>", "lala"},

  182. 			{"<table style=\"FONT-SIZE: 0px;\"><tbody><tr><td>\n"

  183. 			 "DONKEY\n"

  184. 			 "</td></tr></tbody></table>",

  185. 			 ""},

  186. 			/* bgcolor propagation */

  187. 			{"<a style=\"display: inline-block; color: #ffffff; background-color: #00aff0;\">\n"

  188. 			 "<span style=\"color: #00aff0;\">F</span>Rev<span style=\"opacity: 1;\"></span></span>ie<span style=\"opacity: 1;\"></span>"

  189. 			 "</span>w<span style=\"color: #00aff0;\">F<span style=\"opacity: 1;\">̹</span></span>",

  190. 			 " Review"},

  191. 			{"<td style=\"color:#ffffff\" bgcolor=\"#005595\">\n"

  192. 			 "hello world\n"

  193. 			 "</td>",

  194. 			 "hello world"},

  195. 			/* Colors */

  196. 			{"goodbye <span style=\"COLOR: rgb(64,64,64)\">cruel</span>"

  197. 			 "<span>world</span>",

  198. 			 "goodbye cruelworld"},

  199. 			/* Font-size propagation */

  200. 			{"<p style=\"font-size: 11pt;line-height:22px\">goodbye <span style=\"font-size:0px\">cruel</span>world</p>",

  201. 			 "goodbye world\n"},

  202. 			/* Newline before tag -> must be space */

  203. 			{"goodbye <span style=\"COLOR: rgb(64,64,64)\">cruel</span>\n"

  204. 			 "<span>world</span>",

  205. 			 "goodbye cruel world"},

  206. 			/* Head tag with some stuff */

  207. 			{"<html><head><p>oh my god</head><body></body></html>", "oh my god\n"},

  208. 			{"<html><head><title>oh my god</head><body></body></html>", ""},

  209. 			{"<html><body><html><head>displayed</body></html></body></html>", "displayed"},

  210. 

  211. 		};

  212. 

  213. 		rspamd_url_init(NULL);

  214. 		auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),

  215. 										"html", 0);

  216. 		struct rspamd_task fake_task;

  217. 		memset(&fake_task, 0, sizeof(fake_task));

  218. 		fake_task.task_pool = pool;

  219. 

  220. 		auto replace_newlines = [](std::string &str) {

  221. 			auto start_pos = 0;

  222. 			while ((start_pos = str.find("\n", start_pos, 1)) != std::string::npos) {

  223. 				str.replace(start_pos, 1, "\\n", 2);

  224. 				start_pos += 2;

  225. 			}

  226. 		};

  227. 

  228. 		auto i = 1;

  229. 		for (const auto &c: cases) {

  230. 			SUBCASE((fmt::format("html extraction case {}", i)).c_str())

  231. 			{

  232. 				GByteArray *tmp = g_byte_array_sized_new(c.first.size());

  233. 				g_byte_array_append(tmp, (const uint8_t *) c.first.data(), c.first.size());

  234. 				auto *hc = html_process_input(&fake_task, tmp, nullptr, nullptr, nullptr, true, nullptr);

  235. 				CHECK(hc != nullptr);

  236. 				replace_newlines(hc->parsed);

  237. 				auto expected = c.second;

  238. 				replace_newlines(expected);

  239. 				CHECK(hc->parsed == expected);

  240. 				g_byte_array_free(tmp, TRUE);

  241. 			}

  242. 			i++;

  243. 		}

  244. 

  245. 		rspamd_mempool_delete(pool);

  246. 	}

  247. 

  248. 	TEST_CASE("html urls extraction")

  249. 	{

  250. 		using namespace std::string_literals;

  251. 		const std::vector<std::tuple<std::string, std::vector<std::string>, std::optional<std::string>>> cases{

  252. 			{"<style></style><a href=\"https://www.example.com\">yolo</a>",

  253. 			 {"https://www.example.com"},

  254. 			 "yolo"},

  255. 			{"<a href=\"https://example.com\">test</a>", {"https://example.com"}, "test"},

  256. 			{"<a <poo href=\"http://example.com\">hello</a>", {"http://example.com"}, "hello"},

  257. 			{"<html>\n"

  258. 			 "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=utf-8\">\n"

  259. 			 "<body>\n"

  260. 			 "<a href=\"https://www.example.com\">hello</a>\n"

  261. 			 "</body>\n"

  262. 			 "</html>",

  263. 			 {"https://www.example.com"},

  264. 			 "hello"},

  265. 		};

  266. 

  267. 		rspamd_url_init(NULL);

  268. 		auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),

  269. 										"html", 0);

  270. 		struct rspamd_task fake_task;

  271. 		memset(&fake_task, 0, sizeof(fake_task));

  272. 		fake_task.task_pool = pool;

  273. 

  274. 		auto i = 1;

  275. 		for (const auto &c: cases) {

  276. 			SUBCASE((fmt::format("html url extraction case {}", i)).c_str())

  277. 			{

  278. 				GPtrArray *purls = g_ptr_array_new();

  279. 				auto input = std::get<0>(c);

  280. 				GByteArray *tmp = g_byte_array_sized_new(input.size());

  281. 				g_byte_array_append(tmp, (const uint8_t *) input.data(), input.size());

  282. 				auto *hc = html_process_input(&fake_task, tmp, nullptr, nullptr, purls, true, nullptr);

  283. 				CHECK(hc != nullptr);

  284. 				auto &expected_text = std::get<2>(c);

  285. 				if (expected_text.has_value()) {

  286. 					CHECK(hc->parsed == expected_text.value());

  287. 				}

  288. 				const auto &expected_urls = std::get<1>(c);

  289. 				CHECK(expected_urls.size() == purls->len);

  290. 				for (auto j = 0; j < expected_urls.size(); ++j) {

  291. 					auto *url = (rspamd_url *) g_ptr_array_index(purls, j);

  292. 					CHECK(expected_urls[j] == std::string{url->string, url->urllen});

  293. 				}

  294. 				g_byte_array_free(tmp, TRUE);

  295. 				g_ptr_array_free(purls, TRUE);

  296. 			}

  297. 			++i;

  298. 		}

  299. 

  300. 		rspamd_mempool_delete(pool);

  301. 	}

  302. }

  303. 

  304. } /* namespace rspamd::html */