* Extract domains from emails to check via surbl

* Fixes to url extracter
* Check urls in redirector even if they are listed in surbl whitelist

2 files changed, 46 insertions, 15 deletions

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 2fbeb5e2b..34280101e 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -307,7 +307,8 @@ surbl_module_reconfig (struct config_file *cfg)
 
 
 static char                    *
-format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_item *suffix, gboolean append_suffix, GError ** err)
+format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_item *suffix,
+		gboolean append_suffix, GError ** err, gboolean forced)
 {
 	GHashTable                     *t;
 	char                           *result = NULL, *dots[MAX_LEVELS], num_buf[sizeof("18446744073709551616")], *p;
@@ -409,7 +410,7 @@ format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_it
 		}
 	}
 
-	if (g_hash_table_lookup (surbl_module_ctx->whitelist, result) != NULL) {
+	if (!forced && g_hash_table_lookup (surbl_module_ctx->whitelist, result) != NULL) {
 		msg_debug ("url %s is whitelisted", result);
 		g_set_error (err, SURBL_ERROR,	/* error domain */
 						WHITELIST_ERROR,	/* error code */
@@ -429,7 +430,8 @@ format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_it
 }
 
 static void
-make_surbl_requests (struct uri *url, struct worker_task *task, GTree * tree, struct suffix_item *suffix)
+make_surbl_requests (struct uri *url, struct worker_task *task, GTree * tree,
+		struct suffix_item *suffix, gboolean forced)
 {
 	char                           *surbl_req;
 	f_str_t                         f;
@@ -440,7 +442,7 @@ make_surbl_requests (struct uri *url, struct worker_task *task, GTree * tree, st
 	f.len = url->hostlen;
 
 	if (check_view (task->cfg->views, suffix->symbol, task)) {
-		if ((surbl_req = format_surbl_request (task->task_pool, &f, suffix, TRUE, &err)) != NULL) {
+		if ((surbl_req = format_surbl_request (task->task_pool, &f, suffix, TRUE, &err, forced)) != NULL) {
 			if (g_tree_lookup (tree, surbl_req) == NULL) {
 				g_tree_insert (tree, surbl_req, surbl_req);
 				param = memory_pool_alloc (task->task_pool, sizeof (struct dns_param));
@@ -590,7 +592,7 @@ memcached_callback (memcached_ctx_t * ctx, memc_error_t error, void *data)
 			param->task->save.saved = 1;
 			process_filters (param->task);
 		}
-		make_surbl_requests (param->url, param->task, param->tree, param->suffix);
+		make_surbl_requests (param->url, param->task, param->tree, param->suffix, FALSE);
 		break;
 	default:
 		return;
@@ -656,7 +658,7 @@ free_redirector_session (void *ud)
 	event_del (&param->ev);
 	close (param->sock);
 	param->task->save.saved--;
-	make_surbl_requests (param->url, param->task, param->tree, param->suffix);
+	make_surbl_requests (param->url, param->task, param->tree, param->suffix, FALSE);
 	if (param->task->save.saved == 0) {
 		/* Call other filters */
 		param->task->save.saved = 1;
@@ -739,7 +741,7 @@ register_redirector_call (struct uri *url, struct worker_task *task, GTree * url
 	if (s == -1) {
 		msg_info ("<%s> cannot create tcp socket failed: %s", task->message_id, strerror (errno));
 		task->save.saved--;
-		make_surbl_requests (url, task, url_tree, suffix);
+		make_surbl_requests (url, task, url_tree, suffix, FALSE);
 		return;
 	}
 
@@ -756,6 +758,8 @@ register_redirector_call (struct uri *url, struct worker_task *task, GTree * url
 	event_set (&param->ev, s, EV_WRITE, redirector_callback, (void *)param);
 	event_add (&param->ev, timeout);
 	register_async_event (task->s, free_redirector_session, param, FALSE);
+
+	msg_info ("<%s> registered redirector call for %s", task->message_id, struri (url));
 }
 
 static                          gboolean
@@ -774,14 +778,14 @@ tree_url_callback (gpointer key, gpointer value, void *data)
 	if (surbl_module_ctx->use_redirector) {
 		f.begin = url->host;
 		f.len = url->hostlen;
-		if ((urlstr = format_surbl_request (param->task->task_pool, &f, NULL, FALSE, &err)) != NULL) {
+		if ((urlstr = format_surbl_request (param->task->task_pool, &f, NULL, FALSE, &err, TRUE)) != NULL) {
 			if (g_hash_table_lookup (surbl_module_ctx->redirector_hosts, urlstr) != NULL) {
 				register_redirector_call (url, param->task, param->tree, param->suffix);
 				param->task->save.saved++;
 				return FALSE;
 			}
 		}
-		make_surbl_requests (url, param->task, param->tree, param->suffix);
+		make_surbl_requests (url, param->task, param->tree, param->suffix, TRUE);
 	}
 	else {
 		if (param->task->worker->srv->cfg->memcached_servers_num > 0) {
@@ -789,7 +793,7 @@ tree_url_callback (gpointer key, gpointer value, void *data)
 			param->task->save.saved++;
 		}
 		else {
-			make_surbl_requests (url, param->task, param->tree, param->suffix);
+			make_surbl_requests (url, param->task, param->tree, param->suffix, FALSE);
 		}
 	}
 
@@ -872,7 +876,7 @@ urls_command_handler (struct worker_task *task)
 			g_tree_insert (url_tree, struri (url), url);
 			f.begin = url->host;
 			f.len = url->hostlen;
-			if ((urlstr = format_surbl_request (task->task_pool, &f, NULL, FALSE, &err)) != NULL) {
+			if ((urlstr = format_surbl_request (task->task_pool, &f, NULL, FALSE, &err, FALSE)) != NULL) {
 				if (g_list_next (cur) != NULL) {
 					r += rspamd_snprintf (outbuf + r, buflen - r - 2, "%s <\"%s\">, ", (char *)urlstr, struri (url));
 				}
diff --git a/src/url.c b/src/url.c
index b839d7566..5a1665674 100644
--- a/src/url.c
+++ b/src/url.c
@@ -105,9 +105,9 @@ static const struct _proto      protocol_backends[] = {
 	{"ftp", 21, NULL, 1, 1, 0, 0},
 	{"http", 80, NULL, 1, 1, 0, 0},
 	{"https", 443, NULL, 1, 1, 0, 1},
-
+	{"mailto", 25, NULL, 1, 1, 0, 0},
 	/* Keep these last! */
-	{NULL, 0, NULL, 0, 0, 1, 0},
+	{NULL, 0, NULL, 0, 0, 1, 0}
 };
 
 /* Convert an ASCII hex digit to the corresponding number between 0
@@ -952,6 +952,8 @@ url_web_start (const gchar *begin, const gchar *end, const gchar *pos, url_match
 			return FALSE;
 		}
 	}
+	match->m_begin = pos;
+
 	return TRUE;
 }
 static gboolean
@@ -1109,12 +1111,37 @@ domain:
 static gboolean
 url_email_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match)
 {
+	/* Check what we have found */
+	if (pos > begin && *pos == '@') {
+		if (is_atom (*(pos - 1)) && is_domain (*(pos + 1))) {
+			match->m_begin = pos + 1;
+			return TRUE;
+		}
+	}
+	else {
+		while (pos < end && is_atom (*pos)) {
+			if (*pos == '@') {
+				match->m_begin = pos + 1;
+				return TRUE;
+			}
+			pos ++;
+		}
+	}
 	return FALSE;
 }
+
 static gboolean
 url_email_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match)
 {
-	return FALSE;
+	const gchar                    *p;
+
+	p = pos + strlen (match->pattern);
+
+	while (p < end && (is_domain (*p) || (*p == '.' && p + 1 < end && is_domain (*(p + 1))))) {
+		p ++;
+	}
+	match->m_len = p - match->m_begin;
+	return TRUE;
 }
 
 void
@@ -1149,7 +1176,7 @@ url_parse_text (memory_pool_t * pool, struct worker_task *task, struct mime_text
 				matcher = &matchers[idx];
 				m.pattern = matcher->pattern;
 				m.prefix = matcher->prefix;
-				if (matcher->start (p, pos, end, &m) && matcher->end (p, pos, end, &m)) {
+				if (matcher->start (p, end, pos, &m) && matcher->end (p, end, pos, &m)) {
 					url_str = memory_pool_alloc (task->task_pool, m.m_len + 1);
 					memcpy (url_str, m.m_begin, m.m_len);
 					url_str[m.m_len] = '\0';