[Test] SURBL resolve ip

13 files changed, 160 insertions, 71 deletions

diff --git a/conf/maps.d/external_ip_spf.inc b/conf/maps.d/external_ip_spf.inc
deleted file mode 100644
index 3fc5c1775..000000000
--- a/conf/maps.d/external_ip_spf.inc
+++ /dev/null
@@ -1 +0,0 @@
-192.168.1.1
\ No newline at end of file
diff --git a/lualib/lua_content/pdf.lua b/lualib/lua_content/pdf.lua
index f29e1e781..741c011bd 100644
--- a/lualib/lua_content/pdf.lua
+++ b/lualib/lua_content/pdf.lua
@@ -386,8 +386,9 @@ local function process_dict(task, pdf, obj, dict)
     lua_util.debugm(N, task, 'process stream dictionary for object %s:%s -> %s',
         obj.major, obj.minor, obj.type)
     local contents = dict.Contents
-    if contents then
-      if type(contents) == 'table' and contents[1] == '%REF%' then
+    if contents and type(contents) == 'table' then
+      if contents[1] == '%REF%' then
+        -- Single reference
         contents = {contents}
       end
       obj.contents = {}
diff --git a/rules/misc.lua b/rules/misc.lua
index 7d3682271..5dcf6ea05 100644
--- a/rules/misc.lua
+++ b/rules/misc.lua
@@ -685,12 +685,14 @@ local check_encrypted_name = rspamd_config:register_symbol{
     local function check_part(part)
       if part:is_multipart() then
         local children = part:get_children() or {}
+        local text_kids = {}
 
         for _,cld in ipairs(children) do
           if cld:is_multipart() then
             check_part(cld)
           elseif cld:is_text() then
             seen_text = true
+            text_kids[#text_kids + 1] = cld
           else
             local type,subtype,_ = cld:get_type_full()
 
@@ -712,6 +714,17 @@ local check_encrypted_name = rspamd_config:register_symbol{
               end
             end
           end
+          if seen_text and seen_encrypted then
+            -- Ensure that our seen text is not really part of pgp #3205
+            for _,tp in ipairs(text_kids) do
+              local t,_ = tp:get_type()
+              seen_text = false -- reset temporary
+              if t and t == 'text' then
+                seen_text = true
+                break
+              end
+            end
+          end
         end
       end
     end
diff --git a/src/libmime/mime_encoding.c b/src/libmime/mime_encoding.c
index 0ba0e0edd..1f130325e 100644
--- a/src/libmime/mime_encoding.c
+++ b/src/libmime/mime_encoding.c
@@ -36,7 +36,7 @@
 #define RSPAMD_CHARSET_FLAG_ASCII (1 << 1)
 
 #define RSPAMD_CHARSET_CACHE_SIZE 32
-#define RSPAMD_CHARSET_MAX_CONTENT 128
+#define RSPAMD_CHARSET_MAX_CONTENT 512
 
 #define SET_PART_RAW(part) ((part)->flags &= ~RSPAMD_MIME_TEXT_PART_FLAG_UTF)
 #define SET_PART_UTF(part) ((part)->flags |= RSPAMD_MIME_TEXT_PART_FLAG_UTF)
@@ -625,28 +625,30 @@ rspamd_mime_charset_utf_check (rspamd_ftok_t *charset,
 		 * corner cases
 		 */
 		if (content_check) {
-			real_charset = rspamd_mime_charset_find_by_content (in,
-					MIN (RSPAMD_CHARSET_MAX_CONTENT, len));
+			if (rspamd_fast_utf8_validate (in, len) != 0) {
+				real_charset = rspamd_mime_charset_find_by_content (in,
+						MIN (RSPAMD_CHARSET_MAX_CONTENT, len));
 
-			if (real_charset) {
+				if (real_charset) {
 
-				if (rspamd_regexp_match (utf_compatible_re,
-						real_charset, strlen (real_charset), TRUE)) {
-					RSPAMD_FTOK_ASSIGN (charset, UTF8_CHARSET);
+					if (rspamd_regexp_match (utf_compatible_re,
+							real_charset, strlen (real_charset), TRUE)) {
+						RSPAMD_FTOK_ASSIGN (charset, UTF8_CHARSET);
 
-					return TRUE;
-				}
-				else {
-					charset->begin = real_charset;
-					charset->len = strlen (real_charset);
+						return TRUE;
+					}
+					else {
+						charset->begin = real_charset;
+						charset->len = strlen (real_charset);
 
-					return FALSE;
+						return FALSE;
+					}
 				}
+
+				rspamd_mime_charset_utf_enforce (in, len);
 			}
 		}
 
-		rspamd_mime_charset_utf_enforce (in, len);
-
 		return TRUE;
 	}
 
diff --git a/src/libserver/html.c b/src/libserver/html.c
index 502fa42fa..a6b037861 100644
--- a/src/libserver/html.c
+++ b/src/libserver/html.c
@@ -2790,7 +2790,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 							goffset old_offset = dest->len;
 
 							if (content_tag) {
-								if (content_tag->content_offset == 0) {
+								if (content_tag->content_length == 0) {
 									content_tag->content_offset = old_offset;
 								}
 							}
@@ -2810,7 +2810,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 							len = p - c;
 
 							if (content_tag) {
-								if (content_tag->content_offset == 0) {
+								if (content_tag->content_length == 0) {
 									content_tag->content_offset = dest->len;
 								}
 
@@ -2831,7 +2831,18 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 								!g_ascii_isspace (dest->data[dest->len - 1])) {
 							g_byte_array_append (dest, " ", 1);
 							if (content_tag) {
-								content_tag->content_length ++;
+								if (content_tag->content_length == 0) {
+									/*
+									 * Special case
+									 * we have a space at the beginning but
+									 * we have no set content_offset
+									 * so we need to do it here
+									 */
+									content_tag->content_offset = dest->len;
+								}
+								else {
+									content_tag->content_length++;
+								}
 							}
 						}
 						save_space = FALSE;
@@ -2845,7 +2856,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 						goffset old_offset = dest->len;
 
 						if (content_tag) {
-							if (content_tag->content_offset == 0) {
+							if (content_tag->content_length == 0) {
 								content_tag->content_offset = dest->len;
 							}
 						}
@@ -2864,7 +2875,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 						len = p - c;
 
 						if (content_tag) {
-							if (content_tag->content_offset == 0) {
+							if (content_tag->content_length == 0) {
 								content_tag->content_offset = dest->len;
 							}
 
@@ -2962,7 +2973,18 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 						g_byte_array_append (dest, "\r\n", 2);
 
 						if (content_tag) {
-							content_tag->content_length += 2;
+							if (content_tag->content_length == 0) {
+								/*
+								 * Special case
+								 * we have a \r\n at the beginning but
+								 * we have no set content_offset
+								 * so we need to do it here
+								 */
+								content_tag->content_offset = dest->len;
+							}
+							else {
+								content_tag->content_length += 2;
+							}
 						}
 					}
 					save_space = FALSE;
@@ -2975,7 +2997,18 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 						g_byte_array_append (dest, "\r\n", 2);
 
 						if (content_tag) {
-							content_tag->content_length += 2;
+							if (content_tag->content_length == 0) {
+								/*
+								 * Special case
+								 * we have a \r\n at the beginning but
+								 * we have no set content_offset
+								 * so we need to get it here
+								 */
+								content_tag->content_offset = dest->len;
+							}
+							else {
+								content_tag->content_length += 2;
+							}
 						}
 					}
 					save_space = FALSE;
diff --git a/src/libserver/url.c b/src/libserver/url.c
index 54cfbf546..c2f59e768 100644
--- a/src/libserver/url.c
+++ b/src/libserver/url.c
@@ -1550,6 +1550,61 @@ rspamd_tld_trie_callback (struct rspamd_multipattern *mp,
 	return 0;
 }
 
+static void
+rspamd_url_regen_from_inet_addr (struct rspamd_url *uri, const void *addr, int af,
+		rspamd_mempool_t *pool)
+{
+	gchar *strbuf, *p;
+	gsize slen = uri->urllen - uri->hostlen;
+	goffset r = 0;
+
+	if (af == AF_INET) {
+		slen += INET_ADDRSTRLEN;
+	}
+	else {
+		slen += INET6_ADDRSTRLEN;
+	}
+
+	/* Allocate new string to build it from IP */
+	strbuf = rspamd_mempool_alloc (pool, slen + 1);
+	r += rspamd_snprintf (strbuf + r, slen - r, "%*s",
+			(gint)(uri->host - uri->string),
+			uri->string);
+	uri->host = strbuf + r;
+	inet_ntop (af, addr, strbuf + r, slen - r + 1);
+	uri->hostlen = strlen (uri->host);
+	r += uri->hostlen;
+	uri->tld = uri->host;
+	uri->tldlen = uri->hostlen;
+	uri->flags |= RSPAMD_URL_FLAG_NUMERIC;
+
+	/* Reconstruct URL */
+	if (uri->datalen > 0) {
+		p = strbuf + r + 1;
+		r += rspamd_snprintf (strbuf + r, slen - r, "/%*s",
+				(gint)uri->datalen,
+				uri->data);
+		uri->data = p;
+	}
+	if (uri->querylen > 0) {
+		p = strbuf + r + 1;
+		r += rspamd_snprintf (strbuf + r, slen - r, "?%*s",
+				(gint)uri->querylen,
+				uri->query);
+		uri->query = p;
+	}
+	if (uri->fragmentlen > 0) {
+		p = strbuf + r + 1;
+		r += rspamd_snprintf (strbuf + r, slen - r, "#%*s",
+				(gint)uri->fragmentlen,
+				uri->fragment);
+		uri->fragment = p;
+	}
+
+	uri->string = strbuf;
+	uri->urllen = r;
+}
+
 static gboolean
 rspamd_url_is_ip (struct rspamd_url *uri, rspamd_mempool_t *pool)
 {
@@ -1577,23 +1632,11 @@ rspamd_url_is_ip (struct rspamd_url *uri, rspamd_mempool_t *pool)
 	}
 
 	if (rspamd_parse_inet_address_ip4 (p, end - p, &in4)) {
-		uri->host = rspamd_mempool_alloc (pool, INET_ADDRSTRLEN + 1);
-		memset (uri->host, 0, INET_ADDRSTRLEN + 1);
-		inet_ntop (AF_INET, &in4, uri->host, INET_ADDRSTRLEN);
-		uri->hostlen = strlen (uri->host);
-		uri->tld = uri->host;
-		uri->tldlen = uri->hostlen;
-		uri->flags |= RSPAMD_URL_FLAG_NUMERIC;
+		rspamd_url_regen_from_inet_addr (uri, &in4, AF_INET, pool);
 		ret = TRUE;
 	}
 	else if (rspamd_parse_inet_address_ip6 (p, end - p, &in6)) {
-		uri->host = rspamd_mempool_alloc (pool, INET6_ADDRSTRLEN + 1);
-		memset (uri->host, 0, INET6_ADDRSTRLEN + 1);
-		inet_ntop (AF_INET6, &in6, uri->host, INET6_ADDRSTRLEN);
-		uri->hostlen = strlen (uri->host);
-		uri->tld = uri->host;
-		uri->tldlen = uri->hostlen;
-		uri->flags |= RSPAMD_URL_FLAG_NUMERIC;
+		rspamd_url_regen_from_inet_addr (uri, &in6, AF_INET6, pool);
 		ret = TRUE;
 	}
 	else {
@@ -1693,26 +1736,16 @@ rspamd_url_is_ip (struct rspamd_url *uri, rspamd_mempool_t *pool)
 		if (check_num) {
 			if (dots <= 4) {
 				memcpy (&in4, &n, sizeof (in4));
-				uri->host = rspamd_mempool_alloc (pool, INET_ADDRSTRLEN + 1);
-				memset (uri->host, 0, INET_ADDRSTRLEN + 1);
-				inet_ntop (AF_INET, &in4, uri->host, INET_ADDRSTRLEN);
-				uri->hostlen = strlen (uri->host);
-				uri->tld = uri->host;
-				uri->tldlen = uri->hostlen;
-				uri->flags |= RSPAMD_URL_FLAG_NUMERIC | RSPAMD_URL_FLAG_OBSCURED;
+				rspamd_url_regen_from_inet_addr (uri, &in4, AF_INET, pool);
+				uri->flags |=  RSPAMD_URL_FLAG_OBSCURED;
 				ret = TRUE;
 			}
 			else if (end - c > (gint) sizeof (buf) - 1) {
 				rspamd_strlcpy (buf, c, end - c + 1);
 
 				if (inet_pton (AF_INET6, buf, &in6) == 1) {
-					uri->host = rspamd_mempool_alloc (pool, INET6_ADDRSTRLEN + 1);
-					memset (uri->host, 0, INET6_ADDRSTRLEN + 1);
-					inet_ntop (AF_INET6, &in6, uri->host, INET6_ADDRSTRLEN);
-					uri->hostlen = strlen (uri->host);
-					uri->tld = uri->host;
-					uri->tldlen = uri->hostlen;
-					uri->flags |= RSPAMD_URL_FLAG_NUMERIC | RSPAMD_URL_FLAG_OBSCURED;
+					rspamd_url_regen_from_inet_addr (uri, &in6, AF_INET6, pool);
+					uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
 					ret = TRUE;
 				}
 			}
diff --git a/src/lua/lua_html.c b/src/lua/lua_html.c
index 43c34797c..c0e07de36 100644
--- a/src/lua/lua_html.c
+++ b/src/lua/lua_html.c
@@ -663,7 +663,7 @@ lua_html_tag_get_content (lua_State *L)
 	struct rspamd_lua_text *t;
 
 	if (ltag) {
-		if (ltag->html && ltag->tag->content_offset && ltag->tag->content_length &&
+		if (ltag->html && ltag->tag->content_length &&
 				ltag->html->parsed->len >= ltag->tag->content_offset + ltag->tag->content_length) {
 			t = lua_newuserdata (L, sizeof (*t));
 			rspamd_lua_setclass (L, "rspamd{text}", -1);
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index ae1a020a1..0b52cfbdc 100644
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -2763,10 +2763,13 @@ lua_util_is_utf_outside_range(lua_State *L)
 				return 1;
 			}
 
-			rspamd_lru_hash_insert(validators, creation_hash_key, validator, 0, 0);
+			rspamd_lru_hash_insert(validators, creation_hash_key, validator,
+					0, 0);
 		}
 
-		ret = uspoof_checkUTF8 (validator, string_to_check, len_of_string, NULL, &uc_err);
+		gint32 pos = 0;
+		ret = uspoof_checkUTF8 (validator, string_to_check, len_of_string, &pos,
+				&uc_err);
 	}
 	else {
 		return luaL_error (L, "invalid arguments");
diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua
index 67350c285..10daa0d2b 100644
--- a/src/plugins/lua/spf.lua
+++ b/src/plugins/lua/spf.lua
@@ -231,11 +231,6 @@ if local_config.external_relay then
   end
 end
 
-if external_ip_spf.map then
-  local lua_maps = require "lua_maps"
-  external_ip_spf.map = lua_maps.map_add_from_ucl(external_ip_spf.map, "radix", "External IP SPF map")
-end
-
 for _,sym in pairs(local_config.symbols) do
   rspamd_config:register_symbol{
     name = sym,
diff --git a/src/worker.c b/src/worker.c
index 4f13db469..7d9550249 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -364,7 +364,7 @@ accept_socket (EV_P_ ev_io *w, int revents)
 		return;
 	}
 
-	session = g_malloc (sizeof (*session));
+	session = g_malloc0 (sizeof (*session));
 	session->magic = G_MAXINT64;
 	session->addr = addr;
 	session->fd = nfd;
diff --git a/test/functional/cases/340_surbl.robot b/test/functional/cases/340_surbl.robot
index 71bd4285a..962bd30f9 100644
--- a/test/functional/cases/340_surbl.robot
+++ b/test/functional/cases/340_surbl.robot
@@ -11,6 +11,12 @@ ${RSPAMD_SCOPE}  Suite
 ${URL_TLD}      ${TESTDIR}/../lua/unit/test_tld.dat
 
 *** Test Cases ***
+SURBL resolve ip
+  ${result} =  Scan Message With Rspamc  ${TESTDIR}/messages/url7.eml
+  Should Contain  ${result.stdout}  URIBL_SBL_CSS (1.00)[example.ru
+  Should Contain  ${result.stdout}  URIBL_XBL (1.00)[example.ru
+  Should Contain  ${result.stdout}  URIBL_PBL (1.00)[example.ru
+
 SURBL Example.com domain
   ${result} =  Scan Message With Rspamc  ${TESTDIR}/messages/url4.eml
   Should Contain  ${result.stdout}  RSPAMD_URIBL
diff --git a/test/functional/configs/dmarc.conf b/test/functional/configs/dmarc.conf
index dac106cc2..644fff0d5 100644
--- a/test/functional/configs/dmarc.conf
+++ b/test/functional/configs/dmarc.conf
@@ -1,2 +1,4 @@
 dmarc { }
-spf { }
\ No newline at end of file
+spf {
+		external_relay = 192.168.1.1;
+}
\ No newline at end of file
diff --git a/test/lua/unit/url.lua b/test/lua/unit/url.lua
index a748c4de8..269b7841b 100644
--- a/test/lua/unit/url.lua
+++ b/test/lua/unit/url.lua
@@ -25,6 +25,8 @@ context("URL check functions", function()
     {"http://user:password@тест2.РФ:18 text", {"тест2.рф", "user"}},
     {"somebody@example.com", {"example.com", "somebody"}},
     {"https://127.0.0.1/abc text", {"127.0.0.1", nil}},
+    {"https:\\\\127.0.0.1/abc text", {"127.0.0.1", nil}},
+    {"https:\\\\127.0.0.1", {"127.0.0.1", nil}},
     {"https://127.0.0.1 text", {"127.0.0.1", nil}},
     {"https://[::1]:1", {"::1", nil}},
     {"https://user:password@[::1]:1", {"::1", nil}},
@@ -54,10 +56,10 @@ context("URL check functions", function()
   end
 
   cases = {
-    {"http://%30%78%63%30%2e%30%32%35%30.01", true, { --0xc0.0250.01
+    {"http:\\\\%30%78%63%30%2e%30%32%35%30.01", true, { --0xc0.0250.01
       host = '192.168.0.1',
     }},
-    {"http://www.google.com/foo?bar=baz#", true, {
+    {"http:/\\www.google.com/foo?bar=baz#", true, {
       host = 'www.google.com', path = 'foo', query = 'bar=baz', tld = 'google.com'
     }},
     {"http://[www.google.com]/", false},
@@ -78,17 +80,17 @@ context("URL check functions", function()
     {"http://0.0xFFFFFF", true, {
       host = '0.255.255.255'
     }},
-    {"http://030052000001", true, {
+    {"http:/\\030052000001", true, {
       host = '192.168.0.1'
     }},
-    {"http://0xc0.052000001", true, {
+    {"http:\\/0xc0.052000001", true, {
       host = '192.168.0.1'
     }},
-    {"http://192.168.0.1.", true, {
-      host = '192.168.0.1'
+    {"http://192.168.0.1.?foo", true, {
+      host = '192.168.0.1', query = 'foo',
     }},
-    {"http://[::eeee:192.168.0.1]", true, {
-      host = '::eeee:c0a8:1'
+    {"http:/\\[::eeee:192.168.0.1]#test", true, {
+      host = '::eeee:c0a8:1', fragment = 'test'
     }},
     {"http://twitter.com#test", true, {
       host = 'twitter.com', fragment = 'test'