Merge pull request #4324 from rspamd/external-maps

[Feature] Support external maps in Rspamd

5 files changed, 88 insertions, 5 deletions

diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c
index a0cbf0dce..e1c6ccf31 100644
--- a/src/libserver/http/http_connection.c
+++ b/src/libserver/http/http_connection.c
@@ -1867,7 +1867,8 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 
 			if (encrypted) {
 				/* TODO: Add proxy support to HTTPCrypt */
-				rspamd_printf_fstring (buf,
+				if (rspamd_http_message_is_standard_port(msg)) {
+					rspamd_printf_fstring(buf,
 						"%s %s HTTP/1.1\r\n"
 						"Connection: %s\r\n"
 						"Host: %s\r\n"
@@ -1878,9 +1879,25 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 						conn_type,
 						host,
 						enclen);
+				}
+				else {
+					rspamd_printf_fstring(buf,
+						"%s %s HTTP/1.1\r\n"
+						"Connection: %s\r\n"
+						"Host: %s:%d\r\n"
+						"Content-Length: %z\r\n"
+						"Content-Type: application/octet-stream\r\n",
+						"POST",
+						"/post",
+						conn_type,
+						host,
+						msg->port,
+						enclen);
+				}
 			}
 			else {
 				if (conn->priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) {
+					/* Write proxied request */
 					if ((msg->flags & RSPAMD_HTTP_FLAG_HAS_HOST_HEADER)) {
 						rspamd_printf_fstring(buf,
 								"%s %s://%s:%d/%V HTTP/1.1\r\n"
@@ -1895,7 +1912,8 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 								bodylen);
 					}
 					else {
-						rspamd_printf_fstring(buf,
+						if (rspamd_http_message_is_standard_port(msg)) {
+							rspamd_printf_fstring(buf,
 								"%s %s://%s:%d/%V HTTP/1.1\r\n"
 								"Connection: %s\r\n"
 								"Host: %s\r\n"
@@ -1908,9 +1926,27 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 								conn_type,
 								host,
 								bodylen);
+						}
+						else {
+							rspamd_printf_fstring(buf,
+								"%s %s://%s:%d/%V HTTP/1.1\r\n"
+								"Connection: %s\r\n"
+								"Host: %s:%d\r\n"
+								"Content-Length: %z\r\n",
+								http_method_str(msg->method),
+								(conn->opts & RSPAMD_HTTP_CLIENT_SSL) ? "https" : "http",
+								host,
+								msg->port,
+								msg->url,
+								conn_type,
+								host,
+								msg->port,
+								bodylen);
+						}
 					}
 				}
 				else {
+					/* Unproxied version */
 					if ((msg->flags & RSPAMD_HTTP_FLAG_HAS_HOST_HEADER)) {
 						rspamd_printf_fstring(buf,
 								"%s %V HTTP/1.1\r\n"
@@ -1922,7 +1958,8 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 								bodylen);
 					}
 					else {
-						rspamd_printf_fstring(buf,
+						if (rspamd_http_message_is_standard_port(msg)) {
+							rspamd_printf_fstring(buf,
 								"%s %V HTTP/1.1\r\n"
 								"Connection: %s\r\n"
 								"Host: %s\r\n"
@@ -1932,6 +1969,20 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 								conn_type,
 								host,
 								bodylen);
+						}
+						else {
+							rspamd_printf_fstring(buf,
+								"%s %V HTTP/1.1\r\n"
+								"Connection: %s\r\n"
+								"Host: %s:%d\r\n"
+								"Content-Length: %z\r\n",
+								http_method_str(msg->method),
+								msg->url,
+								conn_type,
+								host,
+								msg->port,
+								bodylen);
+						}
 					}
 				}
 
diff --git a/src/libserver/http/http_message.c b/src/libserver/http/http_message.c
index 23ff85cd7..435cdcf13 100644
--- a/src/libserver/http/http_message.c
+++ b/src/libserver/http/http_message.c
@@ -720,4 +720,14 @@ rspamd_http_message_get_http_host (struct rspamd_http_message *msg,
 	}
 
 	return NULL;
+}
+
+bool
+rspamd_http_message_is_standard_port(struct rspamd_http_message *msg)
+{
+	if (msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) {
+		return msg->port == 443;
+	}
+
+	return msg->port == 80;
 }
\ No newline at end of file
diff --git a/src/libserver/http/http_message.h b/src/libserver/http/http_message.h
index 38f599048..f0c0cc2dc 100644
--- a/src/libserver/http/http_message.h
+++ b/src/libserver/http/http_message.h
@@ -239,6 +239,13 @@ guint rspamd_http_message_get_flags (struct rspamd_http_message *msg);
 const gchar* rspamd_http_message_get_http_host (struct rspamd_http_message *msg,
 		gsize *hostlen);
 
+/**
+ * Returns true if a message has standard port (80 or 443 for https)
+ * @param msg
+ * @return
+ */
+bool rspamd_http_message_is_standard_port(struct rspamd_http_message *msg);
+
 #ifdef  __cplusplus
 }
 #endif
diff --git a/src/libserver/hyperscan_tools.cxx b/src/libserver/hyperscan_tools.cxx
index 96366067d..2499b21ec 100644
--- a/src/libserver/hyperscan_tools.cxx
+++ b/src/libserver/hyperscan_tools.cxx
@@ -147,11 +147,23 @@ public:
 	}
 
 	void add_cached_file(const char *fname) {
-
 		auto mut_fname = std::string{fname};
 		std::size_t sz;
+
 		rspamd_normalize_path_inplace(mut_fname.data(), mut_fname.size(), &sz);
 		mut_fname.resize(sz);
+
+		if (mut_fname.empty()) {
+			msg_err_hyperscan("attempt to add an empty hyperscan file!");
+			return;
+		}
+
+		if (access(mut_fname.c_str(), R_OK) == -1) {
+			msg_err_hyperscan("attempt to add non existing hyperscan file: %s, %s", mut_fname.c_str(),
+				strerror(errno));
+			return;
+		}
+
 		auto dir = hs_known_files_cache::get_dir(mut_fname);
 		auto ext =  hs_known_files_cache::get_extension(mut_fname);
 
diff --git a/src/libserver/url.c b/src/libserver/url.c
index 932cd9e85..4984b0d2d 100644
--- a/src/libserver/url.c
+++ b/src/libserver/url.c
@@ -284,6 +284,7 @@ struct url_match_scanner {
 	GArray *matchers_strict;
 	struct rspamd_multipattern *search_trie_full;
 	struct rspamd_multipattern *search_trie_strict;
+	bool has_tld_file;
 };
 
 struct url_match_scanner *url_scanner = NULL;
@@ -602,10 +603,12 @@ rspamd_url_init (const gchar *tld_file)
 				sizeof (struct url_matcher), 13000);
 		url_scanner->search_trie_full = rspamd_multipattern_create_sized (13000,
 				RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8);
+		url_scanner->has_tld_file = true;
 	}
 	else {
 		url_scanner->matchers_full = NULL;
 		url_scanner->search_trie_full = NULL;
+		url_scanner->has_tld_file = false;
 	}
 
 	rspamd_url_add_static_matchers (url_scanner);
@@ -2490,7 +2493,7 @@ rspamd_url_parse (struct rspamd_url *uri,
 
 		if (uri->tldlen == 0) {
 			if (uri->protocol != PROTOCOL_MAILTO) {
-				if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+				if (url_scanner->has_tld_file && !(parse_flags & RSPAMD_URL_PARSE_HREF)) {
 					/* Ignore URL's without TLD if it is not a numeric URL */
 					if (!rspamd_url_is_ip(uri, pool)) {
 						return URI_ERRNO_TLD_MISSING;