title

", "+html;++div;+++div;"}, - {"

", "+p;++p;+++a;"}, - {"

", "+div;++a;"}, - /* Broken, as I don't know how the hell this should be really parsed */ - //{"", - // "+html;++xml;++body;+++head;+++body;"} - }; - - rspamd_url_init(NULL); - auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "html", 0); - - for (const auto &c : cases) { - SUBCASE((std::string("extract tags from: ") + c.first).c_str()) { - GByteArray *tmp = g_byte_array_sized_new(c.first.size()); - g_byte_array_append(tmp, (const guint8 *) c.first.data(), c.first.size()); - auto *hc = html_process_input(pool, tmp, nullptr, nullptr, nullptr, true); - CHECK(hc != nullptr); - auto dump = html_debug_structure(*hc); - CHECK(c.second == dump); - g_byte_array_free(tmp, TRUE); - } - } - - rspamd_mempool_delete(pool); } -TEST_CASE("html text extraction") -{ - using namespace std::string_literals; - const std::vector> cases{ - {"test", "test"}, - {"test\0"s, "test\uFFFD"s}, - {"test\0test"s, "test\uFFFDtest"s}, - {"test\0\0test"s, "test\uFFFD\uFFFDtest"s}, - {"test ", "test"}, - {"test foo, bar", "test foo, bar"}, - {"

text

", "text\n"}, - {"olo

text

lolo", "olo\ntext\nlolo"}, - {"

foo

bar

", "foo\nbar\n"}, - {"foobarbaz", "foobarbaz"}, - {"foobarbaz", "foobarbaz"}, - {"foo
baz", "foo\nbaz"}, - {"test", "test"}, - {" test

", "test"}, - {" \n" - " \n" - " a b a > b a < b a & b 'a "a"\n" - " ", R"|(a b a > b a < b a & b 'a "a")|"}, - /* XML tags */ - {"\n" - " \n" - "test", "test"}, - {"" - " \n" - "

\n" - "

\n" - "
\n" - " test

" - "", "\n\n\ntest\n"}, - {"

file " - "sharing

", "fish\n"}, - /* FIXME: broken until rework of css parser */ - //{"

file " - // "sharing

foo", "fish\nfoo"}, - /* Complex html with bad tags */ - {"\n" - "\n" - " \n" - " \n" - " title\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " Hello, world! test\n" - "

data<>\n" - "

\n" - " stuff
?\n" - " \n" - "", "Hello, world! test \ndata<>\nstuff?"}, - {"
test

", "test\n"}, - /* Tables */ - {"\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "
heada headb
data1 data2
", "heada headb\ndata1 data2\n"}, - /* Invalid closing br and hr + comment */ - {" \n" - " \n" - " Hello, world!
test

contentmore content
\n" - "
\n" - " content inside div\n" - "
\n" - " ", "Hello, world!\ntest\ncontentmore content\ncontent inside div\n"}, - /* First closing tag */ - {"\n" - "\n" - "
Hello. I have some bad news.\n" - "

test
\n" - "\n" - "", "Hello. I have some bad news. \n\n\n\n\n\n\n\n\n\n\n\ntest\n"}, - /* Invalid tags */ - {"lol omg oh my!\n" - "words words goodbye","lol omg oh my! words words goodbye"}, - /* Invisible stuff */ - {"
\n" - "
\n" - "FSincerely,
\n" - "
\n" - "8SkypeFWebF
\n" - "kreyes\n" - "
\n" - "
", - " Sincerely,\n Skype Web\n"}, - {"lala", "lala"}, - /* bgcolor propagation */ - {"\n" - "FRevie" - "wFÌ¹", - " Review"}, - {"\n" - "hello world\n" - "", "hello world"}, - /* Colors */ - {"goodbye cruel" - "world", "goodbye cruelworld"}, - /* Font-size propagation */ - {"
goodbye cruelworld
", - "goodbye world\n"}, - /* Newline before tag -> must be space */ - {"goodbye cruel\n" - "world", "goodbye cruel world"}, - /* Head tag with some stuff */ - {"
oh my god", "oh my god\n"}, - {"oh my god</head><body></body></html>", ""}, - }; - - rspamd_url_init(NULL); - auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "html", 0); - - auto replace_newlines = [](std::string &str) { - auto start_pos = 0; - while((start_pos = str.find("\n", start_pos, 1)) != std::string::npos) { - str.replace(start_pos, 1, "\\n", 2); - start_pos += 2; - } - }; - - auto i = 1; - for (const auto &c : cases) { - SUBCASE((fmt::format("html extraction case {}", i)).c_str()) { - GByteArray *tmp = g_byte_array_sized_new(c.first.size()); - g_byte_array_append(tmp, (const guint8 *) c.first.data(), c.first.size()); - auto *hc = html_process_input(pool, tmp, nullptr, nullptr, nullptr, true); - CHECK(hc != nullptr); - replace_newlines(hc->parsed); - auto expected = c.second; - replace_newlines(expected); - CHECK(hc->parsed == expected); - g_byte_array_free(tmp, TRUE); - } - i ++; - } - - rspamd_mempool_delete(pool); -} - -} - -} /* namespace rspamd::html */ - void * rspamd_html_process_part_full(rspamd_mempool_t *pool, GByteArray *in, GList **exceptions, diff --git a/src/libserver/html/html.hxx b/src/libserver/html/html.hxx index 5b5d0ddc0..71399d27d 100644 --- a/src/libserver/html/html.hxx +++ b/src/libserver/html/html.hxx @@ -27,6 +27,7 @@ #include <vector> #include <memory> +#include <string> #include "function2/function2.hpp" namespace rspamd::css { @@ -126,6 +127,13 @@ private: auto html_tag_by_name(const std::string_view &name) -> std::optional<tag_id_t>; +auto html_process_input(rspamd_mempool_t *pool, + GByteArray *in, + GList **exceptions, + khash_t (rspamd_url_hash) *url_set, + GPtrArray *part_urls, + bool allow_css) -> html_content *; +auto html_debug_structure(const html_content &hc) -> std::string; } diff --git a/src/libserver/html/html_tests.cxx b/src/libserver/html/html_tests.cxx new file mode 100644 index 000000000..323858d71 --- /dev/null +++ b/src/libserver/html/html_tests.cxx @@ -0,0 +1,223 @@ +/*- + * Copyright 2021 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "html.hxx" + +#include <vector> +#include <fmt/core.h> + + +#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL +#include "doctest/doctest.h" + +namespace rspamd::html { + +/* + * Tests part + */ + +TEST_SUITE("html") { +TEST_CASE("html parsing") +{ + + const std::vector<std::pair<std::string, std::string>> cases{ + {"<html><!DOCTYPE html><body>", "+html;++xml;++body;"}, + {"<html><div><div></div></div></html>", "+html;++div;+++div;"}, + {"<html><div><div></div></html>", "+html;++div;+++div;"}, + {"<html><div><div></div></html></div>", "+html;++div;+++div;"}, + {"<p><p><a></p></a></a>", "+p;++p;+++a;"}, + {"<div><a href=\"http://example.com\"></div></a>", "+div;++a;"}, + /* Broken, as I don't know how the hell this should be really parsed */ + //{"<html><!DOCTYPE html><body><head><body></body></html></body></html>", + // "+html;++xml;++body;+++head;+++body;"} + }; + + rspamd_url_init(NULL); + auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), + "html", 0); + + for (const auto &c : cases) { + SUBCASE((std::string("extract tags from: ") + c.first).c_str()) { + GByteArray *tmp = g_byte_array_sized_new(c.first.size()); + g_byte_array_append(tmp, (const guint8 *) c.first.data(), c.first.size()); + auto *hc = html_process_input(pool, tmp, nullptr, nullptr, nullptr, true); + CHECK(hc != nullptr); + auto dump = html_debug_structure(*hc); + CHECK(c.second == dump); + g_byte_array_free(tmp, TRUE); + } + } + + rspamd_mempool_delete(pool); +} + +TEST_CASE("html text extraction") +{ + using namespace std::string_literals; + const std::vector<std::pair<std::string, std::string>> cases{ + {"test", "test"}, + {"test\0"s, "test\uFFFD"s}, + {"test\0test"s, "test\uFFFDtest"s}, + {"test\0\0test"s, "test\uFFFD\uFFFDtest"s}, + {"test ", "test"}, + {"test foo, bar", "test foo, bar"}, + {"<p>text</p>", "text\n"}, + {"olo<p>text</p>lolo", "olo\ntext\nlolo"}, + {"<div>foo</div><div>bar</div>", "foo\nbar\n"}, + {"<b>foo<i>bar</b>baz</i>", "foobarbaz"}, + {"<b>foo<i>bar</i>baz</b>", "foobarbaz"}, + {"foo<br>baz", "foo\nbaz"}, + {"<a href=https://example.com>test</a>", "test"}, + {"<img alt=test>", "test"}, + {" <body>\n" + " \n" + " a b a > b a < b a & b 'a "a"\n" + " </body>", R"|(a b a > b a < b a & b 'a "a")|"}, + /* XML tags */ + {"<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n" + " <!DOCTYPE html\n" + " PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n" + " \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n" + "<body>test</body>", "test"}, + {"<html><head><meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\"></head>" + " <body>\n" + " <p><br>\n" + " </p>\n" + " <div class=\"moz-forward-container\"><br>\n" + " <br>\n" + " test</div>" + "</body>", "\n\n\ntest\n"}, + {"<div>fi<span style=\"FONT-SIZE: 0px\">le </span>" + "sh<span style=\"FONT-SIZE: 0px\">aring </span></div>", "fish\n"}, + /* FIXME: broken until rework of css parser */ + //{"<div>fi<span style=\"FONT-SIZE: 0px\">le </span>" + // "sh<span style=\"FONT-SIZE: 0px\">aring </div>foo</span>", "fish\nfoo"}, + /* Complex html with bad tags */ + {"<!DOCTYPE html>\n" + "<html lang=\"en\">\n" + " <head>\n" + " <meta charset=\"utf-8\">\n" + " <title>title\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " Hello, world! test\n" + "
data<>\n" + "
\n" + " stuff
?\n" + " \n" + "", "Hello, world! test \ndata<>\nstuff?"}, + {"
test

", "test\n"}, + /* Tables */ + {"\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "
heada headb
data1 data2
", "heada headb\ndata1 data2\n"}, + /* Invalid closing br and hr + comment */ + {" \n" + " \n" + " Hello, world!
test

contentmore content
\n" + "
\n" + " content inside div\n" + "
\n" + " ", "Hello, world!\ntest\ncontentmore content\ncontent inside div\n"}, + /* First closing tag */ + {"\n" + "\n" + "
Hello. I have some bad news.\n" + "

test
\n" + "\n" + "", "Hello. I have some bad news. \n\n\n\n\n\n\n\n\n\n\n\ntest\n"}, + /* Invalid tags */ + {"lol omg oh my!\n" + "words words goodbye","lol omg oh my! words words goodbye"}, + /* Invisible stuff */ + {"\n" + "
\n" + "FSincerely,
\n" + "
\n" + "8SkypeFWebF
\n" + "kreyes\n" + "
\n" + "
", + " Sincerely,\n Skype Web\n"}, + {"lala", "lala"}, + /* bgcolor propagation */ + {"\n" + "FRevie" + "wFÌ¹", + " Review"}, + {"\n" + "hello world\n" + "", "hello world"}, + /* Colors */ + {"goodbye cruel" + "world", "goodbye cruelworld"}, + /* Font-size propagation */ + {"
goodbye cruelworld
", + "goodbye world\n"}, + /* Newline before tag -> must be space */ + {"goodbye cruel\n" + "world", "goodbye cruel world"}, + /* Head tag with some stuff */ + {"
oh my god", "oh my god\n"}, + {"oh my god</head><body></body></html>", ""}, + }; + + rspamd_url_init(NULL); + auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), + "html", 0); + + auto replace_newlines = [](std::string &str) { + auto start_pos = 0; + while((start_pos = str.find("\n", start_pos, 1)) != std::string::npos) { + str.replace(start_pos, 1, "\\n", 2); + start_pos += 2; + } + }; + + auto i = 1; + for (const auto &c : cases) { + SUBCASE((fmt::format("html extraction case {}", i)).c_str()) { + GByteArray *tmp = g_byte_array_sized_new(c.first.size()); + g_byte_array_append(tmp, (const guint8 *) c.first.data(), c.first.size()); + auto *hc = html_process_input(pool, tmp, nullptr, nullptr, nullptr, true); + CHECK(hc != nullptr); + replace_newlines(hc->parsed); + auto expected = c.second; + replace_newlines(expected); + CHECK(hc->parsed == expected); + g_byte_array_free(tmp, TRUE); + } + i ++; + } + + rspamd_mempool_delete(pool); +} + +} + +} /* namespace rspamd::html */ +