* Make surbl checks working

5 files changed, 95 insertions, 70 deletions

diff --git a/rspamd.conf.sample b/rspamd.conf.sample
index 24bcda598..794638a5e 100644
--- a/rspamd.conf.sample
+++ b/rspamd.conf.sample
@@ -46,7 +46,7 @@ logging {
 
 # Limit for statfile pool size
 # Default: 100M
-statfile_pool_size = 20M;
+statfile_pool_size = 40M;
 
 
 # Sample statfile definition
@@ -69,6 +69,8 @@ statfile {
 statfile {
 	alias = "test.ham";
 	pattern = "./test.ham";
-	weight = -1.0;
+	weight = -2.0;
 	size = 10M;
 };
+
+url_filters = "surbl";
diff --git a/src/cfg_utils.c b/src/cfg_utils.c
index 310c54fd2..99e7881b8 100644
--- a/src/cfg_utils.c
+++ b/src/cfg_utils.c
@@ -394,13 +394,14 @@ parse_filters_str (struct config_file *cfg, const char *str, enum script_type ty
 	}
 
 	p = strvec;
-	while (*p++) {
+	while (*p) {
 		cur = NULL;
 		/* Search modules from known C modules */
 		for (i = 0; i < MODULES_NUM; i++) {
 			if (strcasecmp (modules[i].name, *p) == 0) {
 				cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct filter));
 				cur->type = C_FILTER;
+				msg_debug ("parse_filters_str: found C filter %s", *p);
 				switch (type) {
 					case SCRIPT_HEADER:
 						cur->func_name = memory_pool_strdup (cfg->cfg_pool, *p);
@@ -424,6 +425,7 @@ parse_filters_str (struct config_file *cfg, const char *str, enum script_type ty
 		}
 		if (cur != NULL) {
 			/* Go to next iteration */
+			p++;
 			continue;
 		}
 		cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct filter));
@@ -446,6 +448,7 @@ parse_filters_str (struct config_file *cfg, const char *str, enum script_type ty
 				LIST_INSERT_HEAD (&cfg->url_filters, cur, next);
 				break;
 		}
+		p ++;
 	}
 
 	g_strfreev (strvec);
diff --git a/src/filter.c b/src/filter.c
index d4953caf2..651aac267 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -77,15 +77,17 @@ factor_consolidation_func (struct worker_task *task, const char *metric_name)
  * Call perl or C module function for specified part of message 
  */
 static void
-call_filter_by_name (struct worker_task *task, const char *name, enum script_type sc_type, enum filter_type filt_type)
+call_filter_by_name (struct worker_task *task, const char *name, enum filter_type filt_type, enum script_type sc_type)
 {
 	struct module_ctx *c_module;
-
+	int res = 0;
+	
 	switch (filt_type) {
 		case C_FILTER:
 			c_module = g_hash_table_lookup (task->worker->srv->cfg->c_modules, name);
 			if (c_module) {
-				switch (filt_type) {
+				res = 1;
+				switch (sc_type) {
 					case SCRIPT_HEADER:
 						c_module->header_filter (task);
 						break;
@@ -100,9 +102,13 @@ call_filter_by_name (struct worker_task *task, const char *name, enum script_typ
 						break;
 				}
 			}
+			else {
+				msg_debug ("call_filter_by_name: %s is not a C module", name);
+			}
 			break;
 		case PERL_FILTER:
-			switch (filt_type) {
+			res = 1;
+			switch (sc_type) {
 				case SCRIPT_HEADER:
 					perl_call_header_filter (name, task);
 					break;
@@ -118,6 +124,8 @@ call_filter_by_name (struct worker_task *task, const char *name, enum script_typ
 			}
 			break;
 	}
+
+	msg_debug ("call_filter_by_name: filter name: %s, result: %d", name, (int)res);
 }
 
 static void
@@ -403,12 +411,12 @@ statfiles_callback (gpointer key, gpointer value, void *arg)
 		if ((res = g_hash_table_lookup (data->metrics, st->metric)) == NULL) {
 			res = memory_pool_alloc (task->task_pool, sizeof (struct statfile_result));
 			res->symbols = g_list_prepend (NULL, st->alias);
-			res->weight = st->classifier->add_result_func (0, weight);
+			res->weight = st->classifier->add_result_func (0, weight * st->weight);
 			g_hash_table_insert (data->metrics, st->metric, res);
 		}
 		else {
 			res->symbols = g_list_prepend (NULL, st->alias);
-			res->weight = st->classifier->add_result_func (res->weight, weight);
+			res->weight = st->classifier->add_result_func (res->weight, weight * st->weight);
 		}
 	}
 	
diff --git a/src/main.c b/src/main.c
index c903d3ede..f315367d7 100644
--- a/src/main.c
+++ b/src/main.c
@@ -227,8 +227,10 @@ main (int argc, char **argv, char **env)
 
 	rspamd->cfg->cfg_name = memory_pool_strdup (rspamd->cfg->cfg_pool, FIXED_CONFIG_FILE);
 	read_cmd_line (argc, argv, rspamd->cfg);
-
-    msg_warn ("(main) starting...");
+    
+    /* First set logger to console logger */
+    cfg->log_fd = 2;
+	g_log_set_default_handler (file_log_function, cfg);
 
 	#ifndef HAVE_SETPROCTITLE
 	init_title (argc, argv, environ);
@@ -236,26 +238,27 @@ main (int argc, char **argv, char **env)
 	
 	f = fopen (rspamd->cfg->cfg_name , "r");
 	if (f == NULL) {
-		msg_warn ("cannot open file: %s", rspamd->cfg->cfg_name );
+		msg_err ("main: cannot open file: %s", rspamd->cfg->cfg_name );
 		return EBADF;
 	}
 	yyin = f;
 
 	if (yyparse() != 0 || yynerrs > 0) {
-		msg_warn ("yyparse: cannot parse config file, %d errors", yynerrs);
+		msg_err ("main: cannot parse config file, %d errors", yynerrs);
 		return EBADF;
 	}
 
 	fclose (f);
+    msg_info ("main: starting...");
 	rspamd->cfg->cfg_name = memory_pool_strdup (rspamd->cfg->cfg_pool, rspamd->cfg->cfg_name );
 
 	/* Strictly set temp dir */
     if (!rspamd->cfg->temp_dir) {
-		msg_warn ("tempdir is not set, trying to use $TMPDIR");
+		msg_warn ("main: tempdir is not set, trying to use $TMPDIR");
 		rspamd->cfg->temp_dir = memory_pool_strdup (rspamd->cfg->cfg_pool, getenv ("TMPDIR"));
 
 		if (!rspamd->cfg->temp_dir) {
-			msg_warn ("$TMPDIR is empty too, using /tmp as default");
+			msg_warn ("main: $TMPDIR is empty too, using /tmp as default");
 	    	rspamd->cfg->temp_dir = memory_pool_strdup (rspamd->cfg->cfg_pool, "/tmp");
 		}
     }
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 07e2376bf..dc790f1bf 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -238,25 +238,25 @@ surbl_module_reconfig (struct config_file *cfg)
 }
 
 static char *
-format_surbl_request (char *hostname) 
+format_surbl_request (memory_pool_t *pool, f_str_t *hostname) 
 {
 	GMatchInfo *info;
-	char *result;
-
-	result = g_malloc (strlen (hostname) + strlen (surbl_module_ctx->suffix) + 1); 
+	char *result = NULL;
+    int len;
+    
+    len = hostname->len + strlen (surbl_module_ctx->suffix) + 2;
 
 	/* First try to match numeric expression */
-	if (g_regex_match (extract_numeric_regexp, hostname, 0, &info) == TRUE) {
+	if (g_regex_match_full (extract_numeric_regexp, hostname->begin, hostname->len, 0, 0, &info, NULL) == TRUE) {
 		gchar *octet1, *octet2, *octet3, *octet4;
-		octet1 = g_match_info_fetch (info, 0);
-		g_match_info_next (info, NULL);
-		octet2 = g_match_info_fetch (info, 0);
-		g_match_info_next (info, NULL);
-		octet3 = g_match_info_fetch (info, 0);
-		g_match_info_next (info, NULL);
-		octet4 = g_match_info_fetch (info, 0);
+		octet1 = g_match_info_fetch (info, 1);
+		octet2 = g_match_info_fetch (info, 2);
+		octet3 = g_match_info_fetch (info, 3);
+		octet4 = g_match_info_fetch (info, 4);
 		g_match_info_free (info);
-		sprintf (result, "%s.%s.%s.%s.%s", octet4, octet3, octet2, octet1, surbl_module_ctx->suffix);
+		result = memory_pool_alloc (pool, len); 
+		msg_debug ("format_surbl_request: got numeric host for check: %s.%s.%s.%s", octet1, octet2, octet3, octet4);
+		snprintf (result, len, "%s.%s.%s.%s.%s", octet4, octet3, octet2, octet1, surbl_module_ctx->suffix);
 		g_free (octet1);
 		g_free (octet2);
 		g_free (octet3);
@@ -265,26 +265,25 @@ format_surbl_request (char *hostname)
 	}
 	g_match_info_free (info);
 	/* Try to match normal domain */
-	if (g_regex_match (extract_normal_regexp, hostname, 0, &info) == TRUE) {
+	if (g_regex_match_full (extract_normal_regexp, hostname->begin, hostname->len, 0, 0, &info, NULL) == TRUE) {
 		gchar *part1, *part2;
-		part1 = g_match_info_fetch (info, 0);
-		g_match_info_next (info, NULL);
-		part2 = g_match_info_fetch (info, 0);
+		part1 = g_match_info_fetch (info, 1);
+		part2 = g_match_info_fetch (info, 2);
 		g_match_info_free (info);
-		sprintf (result, "%s.%s", part1, part2);
+		result = memory_pool_alloc (pool, len); 
+		msg_debug ("format_surbl_request: got normal 2-d level domain %s.%s", part1, part2);
 		if (g_hash_table_lookup (surbl_module_ctx->hosters, result) != NULL) {
 			/* Match additional part for hosters */
 			g_free (part1);
 			g_free (part2);
-			if (g_regex_match (extract_hoster_regexp, hostname, 0, &info) == TRUE) {
+			if (g_regex_match_full (extract_hoster_regexp, hostname->begin, hostname->len, 0, 0, &info, NULL) == TRUE) {
 				gchar *hpart1, *hpart2, *hpart3;
-				hpart1 = g_match_info_fetch (info, 0);
-				g_match_info_next (info, NULL);
-				hpart2 = g_match_info_fetch (info, 0);
-				g_match_info_next (info, NULL);
-				hpart3 = g_match_info_fetch (info, 0);
+				hpart1 = g_match_info_fetch (info, 1);
+				hpart2 = g_match_info_fetch (info, 2);
+				hpart3 = g_match_info_fetch (info, 3);
 				g_match_info_free (info);
-				sprintf (result, "%s.%s.%s.%s", hpart1, hpart2, hpart3, surbl_module_ctx->suffix);
+				msg_debug ("format_surbl_request: got hoster 3-d level domain %s.%s.%s", hpart1, hpart2, hpart3);
+				snprintf (result, len, "%s.%s.%s.%s", hpart1, hpart2, hpart3, surbl_module_ctx->suffix);
 				g_free (hpart1);
 				g_free (hpart2);
 				g_free (hpart3);
@@ -292,6 +291,7 @@ format_surbl_request (char *hostname)
 			}
 			return NULL;
 		}
+		snprintf (result, len, "%s.%s.%s", part1, part2, surbl_module_ctx->suffix);
 		g_free (part1);
 		g_free (part2);
 		return result;
@@ -305,12 +305,14 @@ dns_callback (int result, char type, int count, int ttl, void *addresses, void *
 {
 	struct memcached_param *param = (struct memcached_param *)data;
 	
+	msg_debug ("dns_callback: in surbl request callback");
 	/* If we have result from DNS server, this url exists in SURBL, so increase score */
-	if (result != DNS_ERR_NONE || type != DNS_IPv4_A) {
+	if (result == DNS_ERR_NONE && type == DNS_IPv4_A) {
 		msg_info ("surbl_check: url %s is in surbl %s", param->url->host, surbl_module_ctx->suffix);
 		insert_result (param->task, surbl_module_ctx->metric, surbl_module_ctx->symbol, 1);
 	}
 	else {
+		msg_debug ("surbl_check: url %s is not in surbl %s", param->url->host, surbl_module_ctx->suffix);
 		insert_result (param->task, surbl_module_ctx->metric, surbl_module_ctx->symbol, 0);
 	}
 
@@ -321,10 +323,6 @@ dns_callback (int result, char type, int count, int ttl, void *addresses, void *
 		process_filters (param->task);
 	}
 
-	g_free (param->ctx->param->buf);
-	g_free (param->ctx->param);
-	g_free (param->ctx);
-	g_free (param);
 }
 
 static void 
@@ -333,6 +331,7 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
 	struct memcached_param *param = (struct memcached_param *)data;
 	int *url_count;
 	char *surbl_req;
+	f_str_t c;
 
 	switch (ctx->op) {
 		case CMD_CONNECT:
@@ -345,10 +344,6 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
 					param->task->save.saved = 1;
 					process_filters (param->task);
 				}
-				g_free (param->ctx->param->buf);
-				g_free (param->ctx->param);
-				g_free (param->ctx);
-				g_free (param);
 			}
 			else {
 				memc_get (param->ctx, param->ctx->param);
@@ -364,10 +359,6 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
 					param->task->save.saved = 1;
 					process_filters (param->task);
 				}
-				g_free (param->ctx->param->buf);
-				g_free (param->ctx->param);
-				g_free (param->ctx);
-				g_free (param);
 			}
 			else {
 				url_count = (int *)param->ctx->param->buf;
@@ -391,15 +382,14 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
 				param->task->save.saved = 1;
 				process_filters (param->task);
 			}
-			if ((surbl_req = format_surbl_request (param->url->host)) != NULL) {
+			c.begin = param->url->host;
+			c.len = param->url->hostlen;
+			if ((surbl_req = format_surbl_request (param->task->task_pool, &c)) != NULL) {
+				msg_debug ("surbl_memcached: send surbl dns request %s", surbl_req);
 				param->task->save.saved ++;
 				evdns_resolve_ipv4 (surbl_req, DNS_QUERY_NO_SEARCH, dns_callback, (void *)param);
 				return;
 			}
-			g_free (param->ctx->param->buf);
-			g_free (param->ctx->param);
-			g_free (param->ctx);
-			g_free (param);
 			break;
 	}
 }
@@ -413,16 +403,14 @@ register_memcached_call (struct uri *url, struct worker_task *task)
 	gchar *sum_str;
 	int *url_count;
 
-	param = g_malloc (sizeof (struct memcached_param));
-	cur_param = g_malloc (sizeof (memcached_param_t));
-	url_count = g_malloc (sizeof (int));
+	param = memory_pool_alloc (task->task_pool, sizeof (struct memcached_param));
+	cur_param = memory_pool_alloc0 (task->task_pool, sizeof (memcached_param_t));
+	url_count = memory_pool_alloc (task->task_pool, sizeof (int));
 
 	param->url = url;
 	param->task = task;
 
-	param->ctx = g_malloc (sizeof (memcached_ctx_t));
-	bzero (param->ctx, sizeof (memcached_ctx_t));
-	bzero (cur_param, sizeof (memcached_param_t));
+	param->ctx = memory_pool_alloc0 (task->task_pool, sizeof (memcached_ctx_t));
 
 	cur_param->buf = (u_char *)url_count;
 	cur_param->bufsize = sizeof (int);
@@ -435,6 +423,10 @@ register_memcached_call (struct uri *url, struct worker_task *task)
 											task->cfg->memcached_servers_num, sizeof (struct memcached_server),
 											time (NULL), task->cfg->memcached_error_time, task->cfg->memcached_dead_time, task->cfg->memcached_maxerrors,
 											cur_param->key, strlen(cur_param->key));
+	if (selected == NULL) {
+		msg_err ("surbl_register_memcached_call: no memcached servers can be selected");
+		return;
+	}
 	param->ctx->callback = memcached_callback;
 	param->ctx->callback_data = (void *)param;
 	param->ctx->protocol = task->cfg->memcached_protocol;
@@ -480,7 +472,6 @@ redirector_callback (int fd, short what, void *arg)
 						param->task->save.saved = 1;
 						process_filters (param->task);
 					}
-					g_free (param);
 					return;
 				}
 				param->state = STATE_READ;
@@ -494,7 +485,6 @@ redirector_callback (int fd, short what, void *arg)
 					param->task->save.saved = 1;
 					process_filters (param->task);
 				}
-				g_free (param);
 			}
 			break;
 		case STATE_READ:
@@ -523,7 +513,6 @@ redirector_callback (int fd, short what, void *arg)
 					param->task->save.saved = 1;
 					process_filters (param->task);
 				}
-				g_free (param);
 			}
 			else {
 				event_del (&param->ev);
@@ -534,7 +523,6 @@ redirector_callback (int fd, short what, void *arg)
 					param->task->save.saved = 1;
 					process_filters (param->task);
 				}
-				g_free (param);
 			}
 			break;
 	}
@@ -571,7 +559,7 @@ register_redirector_call (struct uri *url, struct worker_task *task)
 			msg_info ("register_redirector_call: connect() failed: %m");
 		}
 	}
-	param = g_malloc (sizeof (struct redirector_param));
+	param = memory_pool_alloc (task->task_pool, sizeof (struct redirector_param));
 	param->url = url;
 	param->task = task;
 	param->state = STATE_READ;
@@ -586,13 +574,34 @@ static int
 surbl_test_url (struct worker_task *task)
 {
 	struct uri *url;
+	char *surbl_req;
+	struct memcached_param *param;
+	f_str_t c;
 
 	TAILQ_FOREACH (url, &task->urls, next) {
+		msg_debug ("surbl_test_url: check url %s", struri (url));
 		if (surbl_module_ctx->use_redirector) {
 			register_redirector_call (url, task);
 		}
 		else {
-			register_memcached_call (url, task);
+			if (task->worker->srv->cfg->memcached_servers_num > 0) {
+				register_memcached_call (url, task);
+			}
+			else {
+				c.begin = url->host;
+				c.len = url->hostlen;
+				if ((surbl_req = format_surbl_request (task->task_pool, &c)) != NULL) {
+					param = memory_pool_alloc (task->task_pool, sizeof (struct memcached_param));
+					param->task = task;
+					param->url = url;
+					msg_debug ("surbl_test_url: send surbl dns request %s", surbl_req);
+					evdns_resolve_ipv4 (surbl_req, DNS_QUERY_NO_SEARCH, dns_callback, (void *)param);
+				}
+				else {
+					msg_info ("surbl_test_url: cannot format url string for surbl %s", struri (url));
+					return;
+				}
+			}
 		}
 		task->save.saved++;
 	}